diff --git a/docs/extensions.md b/docs/extensions.md index b8960f03cd..8a48f1e10d 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -36,6 +36,9 @@ extensions: # This can be whatever name you'd like. The name itself # isn't used by rpm-ostree. sooper-dooper-tracers: + # Optional; defaults to `os-extension`. An OS extension + # is an extension intended to be `rpm-ostree install`ed. + kind: os-extension # List of packages for this extension packages: - strace @@ -47,6 +50,11 @@ extensions: - x86_64 - aarch64 kernel-dev: + # A development extension lists packages useful for + # developing for the target OSTree, but won't be layered + # on top. A common example is kernel modules. No + # depsolving happens, packages listed are downloaded. + kind: development packages: - kernel-devel - kernel-headers diff --git a/rust/src/extensions.rs b/rust/src/extensions.rs index 5abccf4708..0df07457a6 100644 --- a/rust/src/extensions.rs +++ b/rust/src/extensions.rs @@ -21,6 +21,8 @@ const RPMOSTREE_EXTENSIONS_STATE_FILE: &str = ".rpm-ostree-state-chksum"; #[serde(rename_all = "kebab-case")] pub struct Extensions { extensions: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + repos: Option>, } #[derive(Serialize, Deserialize, Debug)] @@ -31,6 +33,21 @@ pub struct Extension { architectures: Option>, #[serde(skip_serializing_if = "Option::is_none")] match_base_evr: Option, + #[serde(default)] + kind: ExtensionKind, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +enum ExtensionKind { + OsExtension, + Development, +} + +impl Default for ExtensionKind { + fn default() -> Self { + ExtensionKind::OsExtension + } } fn extensions_load_stream( @@ -53,9 +70,11 @@ fn extensions_load_stream( .collect(); for (_, ext) in parsed.extensions.iter_mut() { - for pkg in &ext.packages { - if base_pkgs.contains_key(pkg.as_str()) { - bail!("package {} already present in base", pkg); + if ext.kind == ExtensionKind::OsExtension { + for pkg in &ext.packages { + if base_pkgs.contains_key(pkg.as_str()) { + bail!("package {} already present in base", pkg); + } } } if let Some(ref matched_base_pkg) = ext.match_base_evr { @@ -78,16 +97,30 @@ pub(crate) fn extensions_load( path: &str, basearch: &str, base_pkgs: &Vec, -) -> Result> { +) -> CxxResult> { let f = utils::open_file(path)?; let mut f = std::io::BufReader::new(f); - extensions_load_stream(&mut f, basearch, base_pkgs).with_context(|| format!("parsing {}", path)) + Ok(extensions_load_stream(&mut f, basearch, base_pkgs) + .with_context(|| format!("parsing {}", path))?) } impl Extensions { - pub(crate) fn get_packages(&self) -> Vec { + pub(crate) fn get_repos(&self) -> Vec { + self.repos.as_ref().map(|v| v.clone()).unwrap_or_default() + } + + pub(crate) fn get_os_extension_packages(&self) -> Vec { self.extensions .iter() + .filter(|(_, ext)| ext.kind == ExtensionKind::OsExtension) + .flat_map(|(_, ext)| ext.packages.iter().cloned()) + .collect() + } + + pub(crate) fn get_development_packages(&self) -> Vec { + self.extensions + .iter() + .filter(|(_, ext)| ext.kind == ExtensionKind::Development) .flat_map(|(_, ext)| ext.packages.iter().cloned()) .collect() } @@ -140,6 +173,8 @@ mod tests { #[test] fn basic() { let buf = r###" +repos: + - my-repo extensions: bazboo: packages: @@ -147,7 +182,9 @@ extensions: "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["bazboo"]); + assert!(extensions.get_repos() == vec!["my-repo"]); + assert!(extensions.get_os_extension_packages() == vec!["bazboo"]); + assert!(extensions.get_development_packages().is_empty()); } #[test] @@ -165,6 +202,21 @@ extensions: } } + #[test] + fn ext_in_devel() { + let buf = r###" +extensions: + foobar: + packages: + - foobar + kind: development +"###; + let mut input = std::io::BufReader::new(buf.as_bytes()); + let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); + assert!(extensions.get_os_extension_packages().is_empty()); + assert!(extensions.get_development_packages() == vec!["foobar"]); + } + #[test] fn basearch_filter() { let buf = r###" @@ -183,10 +235,10 @@ extensions: "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["bazboo"]); + assert!(extensions.get_os_extension_packages() == vec!["bazboo"]); let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "s390x", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["dodo", "dada"]); + assert!(extensions.get_os_extension_packages() == vec!["dodo", "dada"]); } #[test] @@ -197,9 +249,16 @@ extensions: packages: - foobar-ext match-base-evr: foobar + kind: os-extension + devel: + packages: + - foobar-devel + match-base-evr: foobar + kind: development "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["foobar-ext-1.2-3"]); + assert!(extensions.get_os_extension_packages() == vec!["foobar-ext-1.2-3"]); + assert!(extensions.get_development_packages() == vec!["foobar-devel-1.2-3"]); } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a7b0240446..df90b3bb8b 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -195,7 +195,9 @@ mod ffi { basearch: &str, base_pkgs: &Vec, ) -> Result>; - fn get_packages(&self) -> Vec; + fn get_repos(&self) -> Vec; + fn get_os_extension_packages(&self) -> Vec; + fn get_development_packages(&self) -> Vec; fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result; fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>; fn serialize_to_dir(&self, output_dir: &str) -> Result<()>; diff --git a/rust/src/treefile.rs b/rust/src/treefile.rs index 20713ccb11..115a9fe655 100644 --- a/rust/src/treefile.rs +++ b/rust/src/treefile.rs @@ -864,9 +864,8 @@ struct TreeComposeConfig { #[serde(skip_serializing_if = "Option::is_none")] #[serde(rename = "add-commit-metadata")] add_commit_metadata: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(rename = "rpmdb")] // The database backend + #[serde(skip_serializing_if = "Option::is_none")] rpmdb: Option, #[serde(flatten)] diff --git a/src/app/rpmostree-compose-builtin-tree.cxx b/src/app/rpmostree-compose-builtin-tree.cxx index 713d238fdd..ab6c27e7a4 100644 --- a/src/app/rpmostree-compose-builtin-tree.cxx +++ b/src/app/rpmostree-compose-builtin-tree.cxx @@ -1532,6 +1532,11 @@ rpmostree_compose_builtin_extensions (int argc, auto extensions = rpmostreecxx::extensions_load (extensions_path, basearch, *packages_mapping); + // notice we don't use a pkgcache repo here like in the treecompose path: we + // want RPMs, so having them already imported isn't useful to us (and anyway, + // for OS extensions by definition they're not expected to be cached since + // they're not in the base tree) + g_autoptr(RpmOstreeContext) ctx = rpmostree_context_new_tree (cachedir_dfd, repo, cancellable, error); if (!ctx) @@ -1556,16 +1561,24 @@ rpmostree_compose_builtin_extensions (int argc, g_autoptr(RpmOstreeTreespec) spec = NULL; { g_autoptr(GPtrArray) gpkgs = g_ptr_array_new_with_free_func (g_free); - auto pkgs = extensions->get_packages(); + auto pkgs = extensions->get_os_extension_packages(); for (auto pkg : pkgs) g_ptr_array_add (gpkgs, (gpointer*) g_strdup (pkg.c_str())); - char **repos = ror_treefile_get_repos (treefile); + + g_autoptr(GPtrArray) grepos = g_ptr_array_new_with_free_func (g_free); + auto repos = extensions->get_repos(); + for (auto repo : repos) + g_ptr_array_add (grepos, (gpointer*) g_strdup (repo.c_str())); + + char **treefile_repos = ror_treefile_get_repos (treefile); + for (char **it = treefile_repos; it && *it; it++) + g_ptr_array_add (grepos, (gpointer*) g_strdup (*it)); + g_autoptr(GKeyFile) treespec = g_key_file_new (); g_key_file_set_string_list (treespec, "tree", "packages", (const char* const*)gpkgs->pdata, gpkgs->len); g_key_file_set_string_list (treespec, "tree", "repos", - (const char* const*)repos, - g_strv_length (repos)); + (const char* const*)grepos->pdata, grepos->len); spec = rpmostree_treespec_new_from_keyfile (treespec, NULL); } @@ -1609,6 +1622,44 @@ rpmostree_compose_builtin_extensions (int argc, return FALSE; } + /* This is hacky: for "development" extensions, we don't want any depsolving + * against the base OS. Rather than awkwardly teach the core about this, we + * just reuse its sack and keep all the functionality here. */ + + DnfContext *dnfctx = rpmostree_context_get_dnf (ctx); + DnfSack *sack = dnf_context_get_sack (dnfctx); + + /* disable the system repo; we always want to download, even if already in the base */ + dnf_sack_repo_enabled (sack, HY_SYSTEM_REPO_NAME, 0); + + auto pkgs = extensions->get_development_packages(); + g_autoptr(GPtrArray) devel_pkgs_to_download = + g_ptr_array_new_with_free_func (g_object_unref); + for (auto pkg : pkgs) + { + g_autoptr(GPtrArray) matches = rpmostree_get_matching_packages (sack, pkg.c_str()); + if (matches->len == 0) + return glnx_throw (error, "Package %s not found", pkg.c_str()); + DnfPackage *found_pkg = (DnfPackage*)matches->pdata[0]; + g_ptr_array_add (devel_pkgs_to_download, g_object_ref (found_pkg)); + } + + rpmostree_set_repos_on_packages (dnfctx, devel_pkgs_to_download); + + if (!rpmostree_download_packages (devel_pkgs_to_download, cancellable, error)) + return FALSE; + + for (guint i = 0; i < devel_pkgs_to_download->len; i++) + { + DnfPackage *pkg = (DnfPackage*)devel_pkgs_to_download->pdata[i]; + const char *src = dnf_package_get_filename (pkg); + const char *basename = glnx_basename (src); + if (!glnx_file_copy_at (AT_FDCWD, dnf_package_get_filename (pkg), NULL, output_dfd, + basename, GLNX_FILE_COPY_NOXATTRS, cancellable, error)) + return FALSE; + } + + // XXX: account for development extensions extensions->update_state_checksum (state_checksum, opt_extensions_output_dir); extensions->serialize_to_dir (opt_extensions_output_dir); if (!process_touch_if_changed (error)) diff --git a/src/libpriv/rpmostree-core.cxx b/src/libpriv/rpmostree-core.cxx index abfb463831..1758cd1070 100644 --- a/src/libpriv/rpmostree-core.cxx +++ b/src/libpriv/rpmostree-core.cxx @@ -1456,6 +1456,36 @@ find_pkg_in_ostree (RpmOstreeContext *self, return TRUE; } +void +rpmostree_set_repos_on_packages (DnfContext *dnfctx, + GPtrArray *packages) +{ + GPtrArray *sources = dnf_context_get_repos (dnfctx); + /* ownership of key and val stays in sources */ + g_autoptr(GHashTable) name_to_repo = g_hash_table_new (g_str_hash, g_str_equal); + for (guint i = 0; i < sources->len; i++) + { + DnfRepo *source = (DnfRepo*)sources->pdata[i]; + g_hash_table_insert (name_to_repo, (gpointer)dnf_repo_get_id (source), source); + } + + for (guint i = 0; i < packages->len; i++) + { + DnfPackage *pkg = (DnfPackage*)packages->pdata[i]; + const char *reponame = dnf_package_get_reponame (pkg); + gboolean is_locally_cached = + (g_strcmp0 (reponame, HY_CMDLINE_REPO_NAME) == 0); + + if (is_locally_cached) + continue; + + DnfRepo *repo = (DnfRepo*)g_hash_table_lookup (name_to_repo, reponame); + g_assert (repo); + + dnf_package_set_repo (pkg, repo); + } +} + /* determine of all the marked packages, which ones we'll need to download, * which ones we'll need to import, and which ones we'll need to relabel */ static gboolean @@ -1475,7 +1505,9 @@ sort_packages (RpmOstreeContext *self, self->pkgs_to_relabel = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); self->n_async_pkgs_relabeled = 0; - GPtrArray *sources = dnf_context_get_repos (dnfctx); + /* make sure all the non-cached pkgs have their repos set */ + rpmostree_set_repos_on_packages (dnfctx, packages); + for (guint i = 0; i < packages->len; i++) { auto pkg = static_cast(packages->pdata[i]); @@ -1487,23 +1519,6 @@ sort_packages (RpmOstreeContext *self, gboolean is_locally_cached = (g_strcmp0 (reponame, HY_CMDLINE_REPO_NAME) == 0); - /* make sure all the non-cached pkgs have their repos set */ - if (!is_locally_cached) - { - DnfRepo *src = NULL; - - /* Hackily look up the source...we need a hash table */ - for (guint j = 0; j < sources->len && !src; j++) - { - auto tmpsrc = static_cast(sources->pdata[j]); - if (g_strcmp0 (reponame, dnf_repo_get_id (tmpsrc)) == 0) - src = tmpsrc; - } - - g_assert (src); - dnf_package_set_repo (pkg, src); - } - /* NB: We're assuming here that the presence of an ostree repo means that * the user intends to import the pkg vs e.g. installing it like during a * treecompose. Even though in the treecompose case, an ostree repo *is* @@ -2428,14 +2443,19 @@ rpmostree_context_get_state_sha512 (RpmOstreeContext *self, } static GHashTable * -gather_source_to_packages (RpmOstreeContext *self) +gather_source_to_packages (GPtrArray *packages) { g_autoptr(GHashTable) source_to_packages = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify)g_ptr_array_unref); - for (guint i = 0; i < self->pkgs_to_download->len; i++) + for (guint i = 0; i < packages->len; i++) { - auto pkg = static_cast(self->pkgs_to_download->pdata[i]); + auto pkg = static_cast(packages->pdata[i]); + + /* ignore local packages */ + if (rpmostree_pkg_is_local (pkg)) + continue; + DnfRepo *src = dnf_package_get_repo (pkg); GPtrArray *source_packages; @@ -2453,6 +2473,39 @@ gather_source_to_packages (RpmOstreeContext *self) return util::move_nullify (source_to_packages); } +gboolean +rpmostree_download_packages (GPtrArray *packages, + GCancellable *cancellable, + GError **error) +{ + guint progress_sigid; + g_autoptr(GHashTable) source_to_packages = gather_source_to_packages (packages); + GLNX_HASH_TABLE_FOREACH_KV (source_to_packages, DnfRepo*, src, GPtrArray*, src_packages) + { + g_autofree char *target_dir = NULL; + glnx_unref_object DnfState *hifstate = dnf_state_new (); + + progress_sigid = g_signal_connect (hifstate, "percentage-changed", + G_CALLBACK (on_hifstate_percentage_changed), + NULL); + g_auto(RpmOstreeProgress) progress = { 0, }; + rpmostree_output_progress_percent_begin (&progress, "Downloading from '%s'", + dnf_repo_get_id (src)); + + target_dir = g_build_filename (dnf_repo_get_location (src), "/packages/", NULL); + if (!glnx_shutil_mkdir_p_at (AT_FDCWD, target_dir, 0755, cancellable, error)) + return FALSE; + + if (!dnf_repo_download_packages (src, src_packages, target_dir, + hifstate, error)) + return FALSE; + + g_signal_handler_disconnect (hifstate, progress_sigid); + } + + return TRUE; +} + gboolean rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, @@ -2470,33 +2523,7 @@ rpmostree_context_download (RpmOstreeContext *self, else return TRUE; - { guint progress_sigid; - g_autoptr(GHashTable) source_to_packages = gather_source_to_packages (self); - GLNX_HASH_TABLE_FOREACH_KV (source_to_packages, DnfRepo*, src, GPtrArray*, src_packages) - { - g_autofree char *target_dir = NULL; - glnx_unref_object DnfState *hifstate = dnf_state_new (); - - progress_sigid = g_signal_connect (hifstate, "percentage-changed", - G_CALLBACK (on_hifstate_percentage_changed), - NULL); - g_auto(RpmOstreeProgress) progress = { 0, }; - rpmostree_output_progress_percent_begin (&progress, "Downloading from '%s'", - dnf_repo_get_id (src)); - - target_dir = g_build_filename (dnf_repo_get_location (src), "/packages/", NULL); - if (!glnx_shutil_mkdir_p_at (AT_FDCWD, target_dir, 0755, cancellable, error)) - return FALSE; - - if (!dnf_repo_download_packages (src, src_packages, target_dir, - hifstate, error)) - return FALSE; - - g_signal_handler_disconnect (hifstate, progress_sigid); - } - } - - return TRUE; + return rpmostree_download_packages (self->pkgs_to_download, cancellable, error); } /* Returns: (transfer none): The rojig package */ diff --git a/src/libpriv/rpmostree-core.h b/src/libpriv/rpmostree-core.h index 54e431a58f..fb347622e0 100644 --- a/src/libpriv/rpmostree-core.h +++ b/src/libpriv/rpmostree-core.h @@ -186,10 +186,17 @@ rpmostree_context_set_vlockmap (RpmOstreeContext *self, GHashTable *map, gboolean strict); +gboolean rpmostree_download_packages (GPtrArray *packages, + GCancellable *cancellable, + GError **error); + gboolean rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, GError **error); +void rpmostree_set_repos_on_packages (DnfContext *dnfctx, + GPtrArray *packages); + gboolean rpmostree_context_execute_rojig (RpmOstreeContext *self, gboolean *out_changed, GCancellable *cancellable, diff --git a/tests/compose.sh b/tests/compose.sh index 1f388262d2..ebe316adfe 100755 --- a/tests/compose.sh +++ b/tests/compose.sh @@ -47,18 +47,49 @@ if [ ! -d compose-cache ]; then rm -rf manifests/ popd # config - if ! has_compose_privileges; then - # Unlike cosa, we don't need as much flexibility since we don't e.g. build - # images. So just create the supermin appliance and root now so each test - # doesn't have to build it. - mkdir -p supermin.{prepare,build} - # we just import the strict minimum here that rpm-ostree needs - rpms="rpm-ostree bash rpm-build coreutils selinux-policy-targeted dhcp-client util-linux" - # shellcheck disable=SC2086 - supermin --prepare --use-installed -o supermin.prepare $rpms - # the reason we do a heredoc here is so that the var substition takes - # place immediately instead of having to proxy them through to the VM - cat > init < config/cache.repo + + pushd config + python3 -c ' +import sys, json +y = json.load(sys.stdin) +y["repos"] = ["cache"] +y["postprocess"] = [] +y.pop("lockfile-repos", None) +json.dump(y, sys.stdout)' < manifest.json > manifest.json.new + mv manifest.json{.new,} + git add . + git -c user.email="composetest@localhost.com" -c user.name="composetest" \ + commit -am 'modifications for tests' + popd # config + + popd # compose-cache +fi + +if ! has_compose_privileges; then + pushd compose-cache + + # Unlike cosa, we don't need as much flexibility since we don't e.g. build + # images. So just create the supermin appliance and root now so each test + # doesn't have to build it. + mkdir -p supermin.{prepare,build} + # we just import the strict minimum here that rpm-ostree needs + rpms="rpm-ostree bash rpm-build coreutils selinux-policy-targeted dhcp-client util-linux" + # shellcheck disable=SC2086 + supermin --prepare --use-installed -o supermin.prepare $rpms + # the reason we do a heredoc here is so that the var substition takes + # place immediately instead of having to proxy them through to the VM + cat > init < config/cache.repo - - pushd config - python3 -c ' -import sys, json -y = json.load(sys.stdin) -y["repos"] = ["cache"] -y["postprocess"] = [] -y.pop("lockfile-repos", None) -json.dump(y, sys.stdout)' < manifest.json > manifest.json.new - mv manifest.json{.new,} - git add . - git -c user.email="composetest@localhost.com" -c user.name="composetest" \ - commit -am 'modifications for tests' - popd # config + chmod a+x init + tar -czf supermin.prepare/init.tar.gz --remove-files init + supermin --build "${fixtures}/supermin.prepare" --size 5G -f ext2 -o supermin.build popd # compose-cache fi diff --git a/tests/compose/test-basic-unified.sh b/tests/compose/test-basic-unified.sh index 0578eb2392..9b511c1856 100755 --- a/tests/compose/test-basic-unified.sh +++ b/tests/compose/test-basic-unified.sh @@ -89,6 +89,16 @@ build_rpm dodo-base build_rpm dodo requires dodo-base build_rpm solitaire +# this is pretty terrible... need --json for `rpm-ostree db list` +kernel_vra=$(rpm-ostree db list --repo=${repo} ${treeref} kernel | tail -n1 | cut -d- -f2-) +kernel_v=$(cut -d- -f1 <<< "$kernel_vra") +kernel_ra=$(cut -d- -f2- <<< "$kernel_vra") +kernel_r=${kernel_ra%.x86_64} + +build_rpm kernel-core version ${kernel_v} release ${kernel_r} +build_rpm kernel-devel version ${kernel_v} release ${kernel_r} +build_rpm kernel-headers version ${kernel_v} release ${kernel_r} + cat > extensions.yaml << EOF extensions: extinct-birds: @@ -100,6 +110,13 @@ extensions: - nonexistent architectures: - badarch + kernel-devel: + kind: development + packages: + - kernel-core + - kernel-devel + - kernel-headers + match-base-evr: kernel EOF # we don't actually need root here, but in CI the cache may be in a qcow2 and @@ -110,10 +127,12 @@ runasroot rpm-ostree compose extensions --repo=${repo} \ --touch-if-changed extensions-changed ls extensions/{dodo-1.0,dodo-base-1.0,solitaire-1.0}-*.rpm +ls extensions/kernel-{core,devel,headers}-${kernel_v}-${kernel_r}.x86_64.rpm test -f extensions-changed assert_jq extensions/extensions.json \ - '.extensions|length == 1' \ - '.extensions["extinct-birds"]' + '.extensions|length == 2' \ + '.extensions["extinct-birds"]' \ + '.extensions["kernel-devel"]' echo "ok extensions" rm extensions-changed