From b611cd30976976fb951bacbba388ea365133fca8 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 17:54:19 -0500 Subject: [PATCH 1/9] core: Factor out function to set repos on pkgs And use a hash table to make it more efficient. Prep for future patch. --- src/libpriv/rpmostree-core.cxx | 51 ++++++++++++++++++++++------------ src/libpriv/rpmostree-core.h | 3 ++ 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/libpriv/rpmostree-core.cxx b/src/libpriv/rpmostree-core.cxx index abfb463831..d98213bd5f 100644 --- a/src/libpriv/rpmostree-core.cxx +++ b/src/libpriv/rpmostree-core.cxx @@ -1456,6 +1456,36 @@ find_pkg_in_ostree (RpmOstreeContext *self, return TRUE; } +void +rpmostree_set_repos_on_packages (DnfContext *dnfctx, + GPtrArray *packages) +{ + GPtrArray *sources = dnf_context_get_repos (dnfctx); + /* ownership of key and val stays in sources */ + g_autoptr(GHashTable) name_to_repo = g_hash_table_new (g_str_hash, g_str_equal); + for (guint i = 0; i < sources->len; i++) + { + DnfRepo *source = (DnfRepo*)sources->pdata[i]; + g_hash_table_insert (name_to_repo, (gpointer)dnf_repo_get_id (source), source); + } + + for (guint i = 0; i < packages->len; i++) + { + DnfPackage *pkg = (DnfPackage*)packages->pdata[i]; + const char *reponame = dnf_package_get_reponame (pkg); + gboolean is_locally_cached = + (g_strcmp0 (reponame, HY_CMDLINE_REPO_NAME) == 0); + + if (is_locally_cached) + continue; + + DnfRepo *repo = (DnfRepo*)g_hash_table_lookup (name_to_repo, reponame); + g_assert (repo); + + dnf_package_set_repo (pkg, repo); + } +} + /* determine of all the marked packages, which ones we'll need to download, * which ones we'll need to import, and which ones we'll need to relabel */ static gboolean @@ -1475,7 +1505,9 @@ sort_packages (RpmOstreeContext *self, self->pkgs_to_relabel = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); self->n_async_pkgs_relabeled = 0; - GPtrArray *sources = dnf_context_get_repos (dnfctx); + /* make sure all the non-cached pkgs have their repos set */ + rpmostree_set_repos_on_packages (dnfctx, packages); + for (guint i = 0; i < packages->len; i++) { auto pkg = static_cast(packages->pdata[i]); @@ -1487,23 +1519,6 @@ sort_packages (RpmOstreeContext *self, gboolean is_locally_cached = (g_strcmp0 (reponame, HY_CMDLINE_REPO_NAME) == 0); - /* make sure all the non-cached pkgs have their repos set */ - if (!is_locally_cached) - { - DnfRepo *src = NULL; - - /* Hackily look up the source...we need a hash table */ - for (guint j = 0; j < sources->len && !src; j++) - { - auto tmpsrc = static_cast(sources->pdata[j]); - if (g_strcmp0 (reponame, dnf_repo_get_id (tmpsrc)) == 0) - src = tmpsrc; - } - - g_assert (src); - dnf_package_set_repo (pkg, src); - } - /* NB: We're assuming here that the presence of an ostree repo means that * the user intends to import the pkg vs e.g. installing it like during a * treecompose. Even though in the treecompose case, an ostree repo *is* diff --git a/src/libpriv/rpmostree-core.h b/src/libpriv/rpmostree-core.h index 54e431a58f..c0dde196ce 100644 --- a/src/libpriv/rpmostree-core.h +++ b/src/libpriv/rpmostree-core.h @@ -190,6 +190,9 @@ gboolean rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, GError **error); +void rpmostree_set_repos_on_packages (DnfContext *dnfctx, + GPtrArray *packages); + gboolean rpmostree_context_execute_rojig (RpmOstreeContext *self, gboolean *out_changed, GCancellable *cancellable, From b1b6304816962a44a2aa69823d49b3e481127705 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 17:58:04 -0500 Subject: [PATCH 2/9] core: Factor out function to download pkgs I want to be able to use this function without an `RpmOstreeContext`. Prep for future patch. --- src/libpriv/rpmostree-core.cxx | 67 +++++++++++++++++++--------------- src/libpriv/rpmostree-core.h | 4 ++ 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/src/libpriv/rpmostree-core.cxx b/src/libpriv/rpmostree-core.cxx index d98213bd5f..8f3742c0b9 100644 --- a/src/libpriv/rpmostree-core.cxx +++ b/src/libpriv/rpmostree-core.cxx @@ -2443,14 +2443,14 @@ rpmostree_context_get_state_sha512 (RpmOstreeContext *self, } static GHashTable * -gather_source_to_packages (RpmOstreeContext *self) +gather_source_to_packages (GPtrArray *packages) { g_autoptr(GHashTable) source_to_packages = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify)g_ptr_array_unref); - for (guint i = 0; i < self->pkgs_to_download->len; i++) + for (guint i = 0; i < packages->len; i++) { - auto pkg = static_cast(self->pkgs_to_download->pdata[i]); + auto pkg = static_cast(packages->pdata[i]); DnfRepo *src = dnf_package_get_repo (pkg); GPtrArray *source_packages; @@ -2468,6 +2468,39 @@ gather_source_to_packages (RpmOstreeContext *self) return util::move_nullify (source_to_packages); } +gboolean +rpmostree_download_packages (GPtrArray *packages, + GCancellable *cancellable, + GError **error) +{ + guint progress_sigid; + g_autoptr(GHashTable) source_to_packages = gather_source_to_packages (packages); + GLNX_HASH_TABLE_FOREACH_KV (source_to_packages, DnfRepo*, src, GPtrArray*, src_packages) + { + g_autofree char *target_dir = NULL; + glnx_unref_object DnfState *hifstate = dnf_state_new (); + + progress_sigid = g_signal_connect (hifstate, "percentage-changed", + G_CALLBACK (on_hifstate_percentage_changed), + NULL); + g_auto(RpmOstreeProgress) progress = { 0, }; + rpmostree_output_progress_percent_begin (&progress, "Downloading from '%s'", + dnf_repo_get_id (src)); + + target_dir = g_build_filename (dnf_repo_get_location (src), "/packages/", NULL); + if (!glnx_shutil_mkdir_p_at (AT_FDCWD, target_dir, 0755, cancellable, error)) + return FALSE; + + if (!dnf_repo_download_packages (src, src_packages, target_dir, + hifstate, error)) + return FALSE; + + g_signal_handler_disconnect (hifstate, progress_sigid); + } + + return TRUE; +} + gboolean rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, @@ -2485,33 +2518,7 @@ rpmostree_context_download (RpmOstreeContext *self, else return TRUE; - { guint progress_sigid; - g_autoptr(GHashTable) source_to_packages = gather_source_to_packages (self); - GLNX_HASH_TABLE_FOREACH_KV (source_to_packages, DnfRepo*, src, GPtrArray*, src_packages) - { - g_autofree char *target_dir = NULL; - glnx_unref_object DnfState *hifstate = dnf_state_new (); - - progress_sigid = g_signal_connect (hifstate, "percentage-changed", - G_CALLBACK (on_hifstate_percentage_changed), - NULL); - g_auto(RpmOstreeProgress) progress = { 0, }; - rpmostree_output_progress_percent_begin (&progress, "Downloading from '%s'", - dnf_repo_get_id (src)); - - target_dir = g_build_filename (dnf_repo_get_location (src), "/packages/", NULL); - if (!glnx_shutil_mkdir_p_at (AT_FDCWD, target_dir, 0755, cancellable, error)) - return FALSE; - - if (!dnf_repo_download_packages (src, src_packages, target_dir, - hifstate, error)) - return FALSE; - - g_signal_handler_disconnect (hifstate, progress_sigid); - } - } - - return TRUE; + return rpmostree_download_packages (self->pkgs_to_download, cancellable, error); } /* Returns: (transfer none): The rojig package */ diff --git a/src/libpriv/rpmostree-core.h b/src/libpriv/rpmostree-core.h index c0dde196ce..fb347622e0 100644 --- a/src/libpriv/rpmostree-core.h +++ b/src/libpriv/rpmostree-core.h @@ -186,6 +186,10 @@ rpmostree_context_set_vlockmap (RpmOstreeContext *self, GHashTable *map, gboolean strict); +gboolean rpmostree_download_packages (GPtrArray *packages, + GCancellable *cancellable, + GError **error); + gboolean rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, GError **error); From f5c689da295c822d725b57e77a96c02fa518126f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 17:59:03 -0500 Subject: [PATCH 3/9] core: Fix handling of local packages when downloading In the core context, this is redundant with `sort_packages` because it won't put local packages in the `pkgs_to_download` array anyway, but we want this check even if we call `rpmostree_download_packages` directly and pass some packages which may be local. --- src/libpriv/rpmostree-core.cxx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libpriv/rpmostree-core.cxx b/src/libpriv/rpmostree-core.cxx index 8f3742c0b9..1758cd1070 100644 --- a/src/libpriv/rpmostree-core.cxx +++ b/src/libpriv/rpmostree-core.cxx @@ -2451,6 +2451,11 @@ gather_source_to_packages (GPtrArray *packages) for (guint i = 0; i < packages->len; i++) { auto pkg = static_cast(packages->pdata[i]); + + /* ignore local packages */ + if (rpmostree_pkg_is_local (pkg)) + continue; + DnfRepo *src = dnf_package_get_repo (pkg); GPtrArray *source_packages; From 1e3a2ebd326185f15d3802ea9b81f22a1acb9490 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 18:02:50 -0500 Subject: [PATCH 4/9] extensions: Support enabling additional repos We want to be able to enable more repos than those in the treefile when downloading extensions. In RHCOS for example, the `kernel-rt` packages come from a separate repo. But also, once we support "development" extensions, we want to support the case where devel packages come from another repo. --- rust/src/extensions.rs | 9 +++++++++ rust/src/lib.rs | 1 + src/app/rpmostree-compose-builtin-tree.cxx | 14 +++++++++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/rust/src/extensions.rs b/rust/src/extensions.rs index 5abccf4708..414efbb862 100644 --- a/rust/src/extensions.rs +++ b/rust/src/extensions.rs @@ -21,6 +21,8 @@ const RPMOSTREE_EXTENSIONS_STATE_FILE: &str = ".rpm-ostree-state-chksum"; #[serde(rename_all = "kebab-case")] pub struct Extensions { extensions: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + repos: Option>, } #[derive(Serialize, Deserialize, Debug)] @@ -85,6 +87,10 @@ pub(crate) fn extensions_load( } impl Extensions { + pub(crate) fn get_repos(&self) -> Vec { + self.repos.as_ref().map(|v| v.clone()).unwrap_or_default() + } + pub(crate) fn get_packages(&self) -> Vec { self.extensions .iter() @@ -140,6 +146,8 @@ mod tests { #[test] fn basic() { let buf = r###" +repos: + - my-repo extensions: bazboo: packages: @@ -147,6 +155,7 @@ extensions: "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); + assert!(extensions.get_repos() == vec!["my-repo"]); assert!(extensions.get_packages() == vec!["bazboo"]); } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a7b0240446..f261f013cc 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -195,6 +195,7 @@ mod ffi { basearch: &str, base_pkgs: &Vec, ) -> Result>; + fn get_repos(&self) -> Vec; fn get_packages(&self) -> Vec; fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result; fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>; diff --git a/src/app/rpmostree-compose-builtin-tree.cxx b/src/app/rpmostree-compose-builtin-tree.cxx index 713d238fdd..b4b2eea80e 100644 --- a/src/app/rpmostree-compose-builtin-tree.cxx +++ b/src/app/rpmostree-compose-builtin-tree.cxx @@ -1559,13 +1559,21 @@ rpmostree_compose_builtin_extensions (int argc, auto pkgs = extensions->get_packages(); for (auto pkg : pkgs) g_ptr_array_add (gpkgs, (gpointer*) g_strdup (pkg.c_str())); - char **repos = ror_treefile_get_repos (treefile); + + g_autoptr(GPtrArray) grepos = g_ptr_array_new_with_free_func (g_free); + auto repos = extensions->get_repos(); + for (auto repo : repos) + g_ptr_array_add (grepos, (gpointer*) g_strdup (repo.c_str())); + + char **treefile_repos = ror_treefile_get_repos (treefile); + for (char **it = treefile_repos; it && *it; it++) + g_ptr_array_add (grepos, (gpointer*) g_strdup (*it)); + g_autoptr(GKeyFile) treespec = g_key_file_new (); g_key_file_set_string_list (treespec, "tree", "packages", (const char* const*)gpkgs->pdata, gpkgs->len); g_key_file_set_string_list (treespec, "tree", "repos", - (const char* const*)repos, - g_strv_length (repos)); + (const char* const*)grepos->pdata, grepos->len); spec = rpmostree_treespec_new_from_keyfile (treespec, NULL); } From c01f862288f68a4dbffe55a0a8202e39ac4b4b0a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 18:05:59 -0500 Subject: [PATCH 5/9] extensions: Fix missing CxxResult --- rust/src/extensions.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/src/extensions.rs b/rust/src/extensions.rs index 414efbb862..de4c886648 100644 --- a/rust/src/extensions.rs +++ b/rust/src/extensions.rs @@ -80,10 +80,11 @@ pub(crate) fn extensions_load( path: &str, basearch: &str, base_pkgs: &Vec, -) -> Result> { +) -> CxxResult> { let f = utils::open_file(path)?; let mut f = std::io::BufReader::new(f); - extensions_load_stream(&mut f, basearch, base_pkgs).with_context(|| format!("parsing {}", path)) + Ok(extensions_load_stream(&mut f, basearch, base_pkgs) + .with_context(|| format!("parsing {}", path))?) } impl Extensions { From 2fee11a9423fce2cf06f487af22fcc68330afd11 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 18:06:58 -0500 Subject: [PATCH 6/9] app/compose: Add comment about pkgcache Gives a bit more info about how the extensions path is different from the base treecompose. --- src/app/rpmostree-compose-builtin-tree.cxx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/app/rpmostree-compose-builtin-tree.cxx b/src/app/rpmostree-compose-builtin-tree.cxx index b4b2eea80e..978ef3bae6 100644 --- a/src/app/rpmostree-compose-builtin-tree.cxx +++ b/src/app/rpmostree-compose-builtin-tree.cxx @@ -1532,6 +1532,11 @@ rpmostree_compose_builtin_extensions (int argc, auto extensions = rpmostreecxx::extensions_load (extensions_path, basearch, *packages_mapping); + // notice we don't use a pkgcache repo here like in the treecompose path: we + // want RPMs, so having them already imported isn't useful to us (and anyway, + // for OS extensions by definition they're not expected to be cached since + // they're not in the base tree) + g_autoptr(RpmOstreeContext) ctx = rpmostree_context_new_tree (cachedir_dfd, repo, cancellable, error); if (!ctx) From 3313a66c704b6aa0a3ed4d0df7d6a96d37a59fda Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 18:08:28 -0500 Subject: [PATCH 7/9] rust/treefile: Drop unnecessary #[serde(rename)] The key is already called `rpmdb`. --- rust/src/treefile.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/src/treefile.rs b/rust/src/treefile.rs index 20713ccb11..115a9fe655 100644 --- a/rust/src/treefile.rs +++ b/rust/src/treefile.rs @@ -864,9 +864,8 @@ struct TreeComposeConfig { #[serde(skip_serializing_if = "Option::is_none")] #[serde(rename = "add-commit-metadata")] add_commit_metadata: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(rename = "rpmdb")] // The database backend + #[serde(skip_serializing_if = "Option::is_none")] rpmdb: Option, #[serde(flatten)] From 609047618b5f1c8ae4b27b07aa73c5a86782d16b Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 18:19:33 -0500 Subject: [PATCH 8/9] extensions: Add support for development extensions In RHCOS, we ship kernel development-related packages as an extension. Those aren't really extensions that are meant to be layered onto the host. They're meant to be used in a build environment somewhere to compile kernel modules. This makes it very different from "OS extensions" in at least two drastic ways: 1. we don't want to do any depsolving (e.g. we don't want to pull in `gcc` or something) 2. some of those packages may be present in the base already, but we still want to redownload them Hesitated putting this functionality in rpm-ostree, but I think in the end it cuts from the benefit of moving this code to rpm-ostree if we can't entirely get rid of the Python script it obsoletes. Plus, being able to use the `match-base-evr` is still really useful for this use case. Let's add a new `kind` key to support this. The traditional extensions are called "OS extensions" and these new extensions are called "development extensions". The latter is not yet part of the state checksum, so change detection doesn't work there. I think that's fine for now though because the primary use case is the kernel, and there we want to match the base version. So if the kernel changes, the base would change too. (Though there's the corner case of adding a new package to the list while at the same version...) --- docs/extensions.md | 8 +++ rust/src/extensions.rs | 65 +++++++++++++++++++--- rust/src/lib.rs | 3 +- src/app/rpmostree-compose-builtin-tree.cxx | 40 ++++++++++++- tests/compose/test-basic-unified.sh | 23 +++++++- 5 files changed, 127 insertions(+), 12 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index b8960f03cd..8a48f1e10d 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -36,6 +36,9 @@ extensions: # This can be whatever name you'd like. The name itself # isn't used by rpm-ostree. sooper-dooper-tracers: + # Optional; defaults to `os-extension`. An OS extension + # is an extension intended to be `rpm-ostree install`ed. + kind: os-extension # List of packages for this extension packages: - strace @@ -47,6 +50,11 @@ extensions: - x86_64 - aarch64 kernel-dev: + # A development extension lists packages useful for + # developing for the target OSTree, but won't be layered + # on top. A common example is kernel modules. No + # depsolving happens, packages listed are downloaded. + kind: development packages: - kernel-devel - kernel-headers diff --git a/rust/src/extensions.rs b/rust/src/extensions.rs index de4c886648..0df07457a6 100644 --- a/rust/src/extensions.rs +++ b/rust/src/extensions.rs @@ -33,6 +33,21 @@ pub struct Extension { architectures: Option>, #[serde(skip_serializing_if = "Option::is_none")] match_base_evr: Option, + #[serde(default)] + kind: ExtensionKind, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +enum ExtensionKind { + OsExtension, + Development, +} + +impl Default for ExtensionKind { + fn default() -> Self { + ExtensionKind::OsExtension + } } fn extensions_load_stream( @@ -55,9 +70,11 @@ fn extensions_load_stream( .collect(); for (_, ext) in parsed.extensions.iter_mut() { - for pkg in &ext.packages { - if base_pkgs.contains_key(pkg.as_str()) { - bail!("package {} already present in base", pkg); + if ext.kind == ExtensionKind::OsExtension { + for pkg in &ext.packages { + if base_pkgs.contains_key(pkg.as_str()) { + bail!("package {} already present in base", pkg); + } } } if let Some(ref matched_base_pkg) = ext.match_base_evr { @@ -92,9 +109,18 @@ impl Extensions { self.repos.as_ref().map(|v| v.clone()).unwrap_or_default() } - pub(crate) fn get_packages(&self) -> Vec { + pub(crate) fn get_os_extension_packages(&self) -> Vec { self.extensions .iter() + .filter(|(_, ext)| ext.kind == ExtensionKind::OsExtension) + .flat_map(|(_, ext)| ext.packages.iter().cloned()) + .collect() + } + + pub(crate) fn get_development_packages(&self) -> Vec { + self.extensions + .iter() + .filter(|(_, ext)| ext.kind == ExtensionKind::Development) .flat_map(|(_, ext)| ext.packages.iter().cloned()) .collect() } @@ -157,7 +183,8 @@ extensions: let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); assert!(extensions.get_repos() == vec!["my-repo"]); - assert!(extensions.get_packages() == vec!["bazboo"]); + assert!(extensions.get_os_extension_packages() == vec!["bazboo"]); + assert!(extensions.get_development_packages().is_empty()); } #[test] @@ -175,6 +202,21 @@ extensions: } } + #[test] + fn ext_in_devel() { + let buf = r###" +extensions: + foobar: + packages: + - foobar + kind: development +"###; + let mut input = std::io::BufReader::new(buf.as_bytes()); + let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); + assert!(extensions.get_os_extension_packages().is_empty()); + assert!(extensions.get_development_packages() == vec!["foobar"]); + } + #[test] fn basearch_filter() { let buf = r###" @@ -193,10 +235,10 @@ extensions: "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["bazboo"]); + assert!(extensions.get_os_extension_packages() == vec!["bazboo"]); let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "s390x", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["dodo", "dada"]); + assert!(extensions.get_os_extension_packages() == vec!["dodo", "dada"]); } #[test] @@ -207,9 +249,16 @@ extensions: packages: - foobar-ext match-base-evr: foobar + kind: os-extension + devel: + packages: + - foobar-devel + match-base-evr: foobar + kind: development "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["foobar-ext-1.2-3"]); + assert!(extensions.get_os_extension_packages() == vec!["foobar-ext-1.2-3"]); + assert!(extensions.get_development_packages() == vec!["foobar-devel-1.2-3"]); } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index f261f013cc..df90b3bb8b 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -196,7 +196,8 @@ mod ffi { base_pkgs: &Vec, ) -> Result>; fn get_repos(&self) -> Vec; - fn get_packages(&self) -> Vec; + fn get_os_extension_packages(&self) -> Vec; + fn get_development_packages(&self) -> Vec; fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result; fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>; fn serialize_to_dir(&self, output_dir: &str) -> Result<()>; diff --git a/src/app/rpmostree-compose-builtin-tree.cxx b/src/app/rpmostree-compose-builtin-tree.cxx index 978ef3bae6..ab6c27e7a4 100644 --- a/src/app/rpmostree-compose-builtin-tree.cxx +++ b/src/app/rpmostree-compose-builtin-tree.cxx @@ -1561,7 +1561,7 @@ rpmostree_compose_builtin_extensions (int argc, g_autoptr(RpmOstreeTreespec) spec = NULL; { g_autoptr(GPtrArray) gpkgs = g_ptr_array_new_with_free_func (g_free); - auto pkgs = extensions->get_packages(); + auto pkgs = extensions->get_os_extension_packages(); for (auto pkg : pkgs) g_ptr_array_add (gpkgs, (gpointer*) g_strdup (pkg.c_str())); @@ -1622,6 +1622,44 @@ rpmostree_compose_builtin_extensions (int argc, return FALSE; } + /* This is hacky: for "development" extensions, we don't want any depsolving + * against the base OS. Rather than awkwardly teach the core about this, we + * just reuse its sack and keep all the functionality here. */ + + DnfContext *dnfctx = rpmostree_context_get_dnf (ctx); + DnfSack *sack = dnf_context_get_sack (dnfctx); + + /* disable the system repo; we always want to download, even if already in the base */ + dnf_sack_repo_enabled (sack, HY_SYSTEM_REPO_NAME, 0); + + auto pkgs = extensions->get_development_packages(); + g_autoptr(GPtrArray) devel_pkgs_to_download = + g_ptr_array_new_with_free_func (g_object_unref); + for (auto pkg : pkgs) + { + g_autoptr(GPtrArray) matches = rpmostree_get_matching_packages (sack, pkg.c_str()); + if (matches->len == 0) + return glnx_throw (error, "Package %s not found", pkg.c_str()); + DnfPackage *found_pkg = (DnfPackage*)matches->pdata[0]; + g_ptr_array_add (devel_pkgs_to_download, g_object_ref (found_pkg)); + } + + rpmostree_set_repos_on_packages (dnfctx, devel_pkgs_to_download); + + if (!rpmostree_download_packages (devel_pkgs_to_download, cancellable, error)) + return FALSE; + + for (guint i = 0; i < devel_pkgs_to_download->len; i++) + { + DnfPackage *pkg = (DnfPackage*)devel_pkgs_to_download->pdata[i]; + const char *src = dnf_package_get_filename (pkg); + const char *basename = glnx_basename (src); + if (!glnx_file_copy_at (AT_FDCWD, dnf_package_get_filename (pkg), NULL, output_dfd, + basename, GLNX_FILE_COPY_NOXATTRS, cancellable, error)) + return FALSE; + } + + // XXX: account for development extensions extensions->update_state_checksum (state_checksum, opt_extensions_output_dir); extensions->serialize_to_dir (opt_extensions_output_dir); if (!process_touch_if_changed (error)) diff --git a/tests/compose/test-basic-unified.sh b/tests/compose/test-basic-unified.sh index 0578eb2392..9b511c1856 100755 --- a/tests/compose/test-basic-unified.sh +++ b/tests/compose/test-basic-unified.sh @@ -89,6 +89,16 @@ build_rpm dodo-base build_rpm dodo requires dodo-base build_rpm solitaire +# this is pretty terrible... need --json for `rpm-ostree db list` +kernel_vra=$(rpm-ostree db list --repo=${repo} ${treeref} kernel | tail -n1 | cut -d- -f2-) +kernel_v=$(cut -d- -f1 <<< "$kernel_vra") +kernel_ra=$(cut -d- -f2- <<< "$kernel_vra") +kernel_r=${kernel_ra%.x86_64} + +build_rpm kernel-core version ${kernel_v} release ${kernel_r} +build_rpm kernel-devel version ${kernel_v} release ${kernel_r} +build_rpm kernel-headers version ${kernel_v} release ${kernel_r} + cat > extensions.yaml << EOF extensions: extinct-birds: @@ -100,6 +110,13 @@ extensions: - nonexistent architectures: - badarch + kernel-devel: + kind: development + packages: + - kernel-core + - kernel-devel + - kernel-headers + match-base-evr: kernel EOF # we don't actually need root here, but in CI the cache may be in a qcow2 and @@ -110,10 +127,12 @@ runasroot rpm-ostree compose extensions --repo=${repo} \ --touch-if-changed extensions-changed ls extensions/{dodo-1.0,dodo-base-1.0,solitaire-1.0}-*.rpm +ls extensions/kernel-{core,devel,headers}-${kernel_v}-${kernel_r}.x86_64.rpm test -f extensions-changed assert_jq extensions/extensions.json \ - '.extensions|length == 1' \ - '.extensions["extinct-birds"]' + '.extensions|length == 2' \ + '.extensions["extinct-birds"]' \ + '.extensions["kernel-devel"]' echo "ok extensions" rm extensions-changed From d39f8801a401535c42cc0c1c890924e09f36e832 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 2 Feb 2021 17:07:17 -0500 Subject: [PATCH 9/9] tests/compose.sh: Always rebuild supermin appliance We always want the latest rpm-ostree binaries tested, so we need to always rerun supermin. Patch better viewed with whitespace ignored. --- tests/compose.sh | 88 +++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/tests/compose.sh b/tests/compose.sh index 1f388262d2..ebe316adfe 100755 --- a/tests/compose.sh +++ b/tests/compose.sh @@ -47,18 +47,49 @@ if [ ! -d compose-cache ]; then rm -rf manifests/ popd # config - if ! has_compose_privileges; then - # Unlike cosa, we don't need as much flexibility since we don't e.g. build - # images. So just create the supermin appliance and root now so each test - # doesn't have to build it. - mkdir -p supermin.{prepare,build} - # we just import the strict minimum here that rpm-ostree needs - rpms="rpm-ostree bash rpm-build coreutils selinux-policy-targeted dhcp-client util-linux" - # shellcheck disable=SC2086 - supermin --prepare --use-installed -o supermin.prepare $rpms - # the reason we do a heredoc here is so that the var substition takes - # place immediately instead of having to proxy them through to the VM - cat > init < config/cache.repo + + pushd config + python3 -c ' +import sys, json +y = json.load(sys.stdin) +y["repos"] = ["cache"] +y["postprocess"] = [] +y.pop("lockfile-repos", None) +json.dump(y, sys.stdout)' < manifest.json > manifest.json.new + mv manifest.json{.new,} + git add . + git -c user.email="composetest@localhost.com" -c user.name="composetest" \ + commit -am 'modifications for tests' + popd # config + + popd # compose-cache +fi + +if ! has_compose_privileges; then + pushd compose-cache + + # Unlike cosa, we don't need as much flexibility since we don't e.g. build + # images. So just create the supermin appliance and root now so each test + # doesn't have to build it. + mkdir -p supermin.{prepare,build} + # we just import the strict minimum here that rpm-ostree needs + rpms="rpm-ostree bash rpm-build coreutils selinux-policy-targeted dhcp-client util-linux" + # shellcheck disable=SC2086 + supermin --prepare --use-installed -o supermin.prepare $rpms + # the reason we do a heredoc here is so that the var substition takes + # place immediately instead of having to proxy them through to the VM + cat > init < config/cache.repo - - pushd config - python3 -c ' -import sys, json -y = json.load(sys.stdin) -y["repos"] = ["cache"] -y["postprocess"] = [] -y.pop("lockfile-repos", None) -json.dump(y, sys.stdout)' < manifest.json > manifest.json.new - mv manifest.json{.new,} - git add . - git -c user.email="composetest@localhost.com" -c user.name="composetest" \ - commit -am 'modifications for tests' - popd # config + chmod a+x init + tar -czf supermin.prepare/init.tar.gz --remove-files init + supermin --build "${fixtures}/supermin.prepare" --size 5G -f ext2 -o supermin.build popd # compose-cache fi