From 54a693a0f0a0830946fa5405f2b391330bb26a68 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 26 Jan 2021 17:45:21 -0500 Subject: [PATCH] Add `cosa buildextend-extensions` This elevates extensions to be "first-class" artifacts. We include a tarball of the extensions in the build dir, as well as details in `meta.json`. This command replaces the `download-extensions` script and leverages rpm-ostree's new support for extension composes: https://github.com/coreos/rpm-ostree/pull/2439 --- mantle/cosa/cosa_v1.go | 10 ++- mantle/cosa/schema_doc.go | 36 ++++++++++ mantle/go.sum | 3 + src/cmd-build | 3 +- src/cmd-buildextend-extensions | 125 +++++++++++++++++++++++++++++++++ src/cmd-fetch | 2 +- src/cmdlib.sh | 28 ++++++-- src/cosalib/cmdlib.py | 11 +++ src/download-extensions | 67 ------------------ src/schema/v1.json | 36 ++++++++++ 10 files changed, 245 insertions(+), 76 deletions(-) create mode 100755 src/cmd-buildextend-extensions delete mode 100755 src/download-extensions diff --git a/mantle/cosa/cosa_v1.go b/mantle/cosa/cosa_v1.go index 1c460fecec..476537de65 100644 --- a/mantle/cosa/cosa_v1.go +++ b/mantle/cosa/cosa_v1.go @@ -1,6 +1,6 @@ package cosa -// generated by "schematyper ../src/schema/v1.json -o cosa/cosa_v1.go.tmp --package=cosa --root-type=Build --ptr-for-omit" -- DO NOT EDIT +// generated by "../tools/bin/schematyper ../src/schema/v1.json -o cosa/cosa_v1.go.tmp --package=cosa --root-type=Build --ptr-for-omit" -- DO NOT EDIT type AliyunImage struct { ImageID string `json:"id"` @@ -39,6 +39,7 @@ type Build struct { CosaDelayedMetaMerge bool `json:"coreos-assembler.delayed-meta-merge,omitempty"` CosaImageChecksum string `json:"coreos-assembler.image-config-checksum,omitempty"` CosaImageVersion int `json:"coreos-assembler.image-genver,omitempty"` + Extensions *Extensions `json:"extensions,omitempty"` FedoraCoreOsParentCommit string `json:"fedora-coreos.parent-commit,omitempty"` FedoraCoreOsParentVersion string `json:"fedora-coreos.parent-version,omitempty"` Gcp *Gcp `json:"gcp,omitempty"` @@ -95,6 +96,13 @@ type Cloudartifact struct { URL string `json:"url"` } +type Extensions struct { + Manifest map[string]interface{} `json:"manifest"` + Path string `json:"path"` + RpmOstreeState string `json:"rpm-ostree-state"` + Sha256 string `json:"sha256"` +} + type Gcp struct { ImageFamily string `json:"family,omitempty"` ImageName string `json:"image"` diff --git a/mantle/cosa/schema_doc.go b/mantle/cosa/schema_doc.go index 8da96b1a56..5445cb9111 100644 --- a/mantle/cosa/schema_doc.go +++ b/mantle/cosa/schema_doc.go @@ -186,6 +186,7 @@ var generatedSchemaJSON = `{ "ibmcloud", "images", "oscontainer", + "extensions", "parent-pkgdiff", "pkgdiff", "release-payload", @@ -524,6 +525,41 @@ var generatedSchemaJSON = `{ "title":"Oscontainer", "$ref": "#/definitions/image" }, + "extensions": { + "$id":"#/properties/extensions", + "type":"object", + "title":"Extensions", + "required": [ + "path", + "sha256", + "rpm-ostree-state", + "manifest" + ], + "properties": { + "path": { + "$id": "#/artifact/Path", + "type":"string", + "title":"Path" + }, + "sha256": { + "$id": "#/artifact/sha256", + "type":"string", + "title":"SHA256" + }, + "rpm-ostree-state": { + "$id":"#/properties/extensions/items/properties/rpm-ostree-state", + "type":"string", + "title":"RpmOstreeState", + "default":"", + "minLength": 64 + }, + "manifest": { + "$id":"#/properties/extensions/items/properties/manifest", + "type":"object", + "title":"Manifest" + } + } + }, "ostree-commit": { "$id":"#/properties/ostree-commit", "type":"string", diff --git a/mantle/go.sum b/mantle/go.sum index f8eff65f39..e4f2115d99 100644 --- a/mantle/go.sum +++ b/mantle/go.sum @@ -287,8 +287,11 @@ github.com/vishvananda/netns v0.0.0-20150710222425-604eaf189ee8/go.mod h1:ZjcWmF github.com/vmware/govmomi v0.15.0/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= diff --git a/src/cmd-build b/src/cmd-build index 2a064e5928..eaa68e1be8 100755 --- a/src/cmd-build +++ b/src/cmd-build @@ -271,7 +271,8 @@ fi if test -n "${previous_commit}"; then extra_compose_args+=(--previous-commit "${previous_commit}") fi -RUNVM_NONET=1 runcompose --cache-only ${FORCE} --add-metadata-from-json "${commitmeta_input_json}" \ +RUNVM_NONET=1 runcompose_tree --cache-only ${FORCE} \ + --add-metadata-from-json "${commitmeta_input_json}" \ --write-composejson-to "${composejson}" \ --ex-write-lockfile-to "${lockfile_out}".tmp \ "${extra_compose_args[@]}" diff --git a/src/cmd-buildextend-extensions b/src/cmd-buildextend-extensions new file mode 100755 index 0000000000..301c2a91c4 --- /dev/null +++ b/src/cmd-buildextend-extensions @@ -0,0 +1,125 @@ +#!/usr/bin/python3 -u + +import argparse +import os +import shutil +import sys + +import createrepo_c as cr + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from cosalib import cmdlib +from cosalib.builds import Builds +from cosalib.meta import GenericBuildMeta + + +def main(): + args = parse_args() + + workdir = os.path.abspath(os.getcwd()) + + builds = Builds() + builddir = builds.get_build_dir(args.build) + buildmeta = GenericBuildMeta(workdir=workdir, build=args.build) + + if 'extensions' in buildmeta and not args.force: + print(f"Extensions already exist: {buildmeta['extensions']['path']}") + print("Use --force to force a rebuild") + return + + treefile_src = 'src/config/manifest.yaml' + extensions_src = 'src/config/extensions.yaml' + if not os.path.exists(extensions_src): + raise Exception(f"Missing {extensions_src}") + + commit = buildmeta['ostree-commit'] + cmdlib.import_ostree_commit('tmp/repo', commit, + buildmeta['images']['ostree']['path']) + + tmpworkdir = prepare_tmpworkdir() + changed = run_rpmostree(tmpworkdir, commit, treefile_src, extensions_src) + if not changed: + # For now, rpm-ostree will always detect a change because we don't seed + # state from the previous build, so we won't hit this. Need to rework + # how change detection is wired in `cmd-build` to do this properly. + return + + outputdir = f"{tmpworkdir}/output" + with open(f'{outputdir}/.rpm-ostree-state-chksum', encoding='utf-8') as f: + rpm_ostree_state_chksum = f.read() + + pkglist = create_yumrepo(outputdir) + extensions_tarball = create_tarball(buildmeta, outputdir, tmpworkdir) + extensions_tarball_base = os.path.basename(extensions_tarball) + + buildmeta['extensions'] = { + "path": extensions_tarball_base, + "sha256": cmdlib.sha256sum_file(extensions_tarball), + "rpm-ostree-state": rpm_ostree_state_chksum, + "manifest": pkglist, + } + + cmdlib.rm_allow_noent(f'{builddir}/{extensions_tarball_base}') + shutil.move(extensions_tarball, builddir) + buildmeta.write(artifact_name='extensions') + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--build", help="Build ID", default='latest') + parser.add_argument("--force", help="Force rebuild", action='store_true') + return parser.parse_args() + + +def prepare_tmpworkdir(): + tmpworkdir = 'tmp/extensions' + if os.path.exists(tmpworkdir): + shutil.rmtree(tmpworkdir) + os.mkdir(tmpworkdir) + return tmpworkdir + + +def run_rpmostree(workdir, commit, treefile, extensions): + cmdlib.cmdlib_sh(f''' + changed_stamp={workdir}/changed + runcompose_extensions {workdir}/output {treefile} {extensions} \ + --base-rev {commit}''') + return os.path.exists(f'{workdir}/changed') + + +def create_yumrepo(repodir): + cmdlib.run_verbose(['createrepo_c', repodir]) + # we could also have rpm-ostree output the pkglist for us, but meh... we + # need to run createrepo_c anyway and it's nice that we're using it as the + # source of truth, since that's what rpm-ostree clients will also use + repomd = cr.Repomd(os.path.join(repodir, "repodata/repomd.xml")) + pkglist = {} + + def cb(pkg): + epoch = '' + if pkg.epoch and int(pkg.epoch) > 0: + epoch = f'{pkg.epoch}:' + pkglist[pkg.name] = f'{epoch}{pkg.version}-{pkg.release}.{pkg.arch}' + + for record in repomd.records: + if record.type == 'primary': + primary_xml = os.path.join(repodir, record.location_href) + cr.xml_parse_primary(primary_xml, do_files=False, pkgcb=cb) + break + + if len(pkglist) == 0: + raise Exception("No RPMs found in output dir") + return pkglist + + +def create_tarball(buildmeta, srcdir, destdir): + destdir = os.path.abspath(destdir) + basearch = buildmeta['coreos-assembler.basearch'] + tarfile = f'{destdir}/{buildmeta["name"]}-{buildmeta["buildid"]}-extensions.{basearch}.tar' + cmdlib.run_verbose(['tar', '-cf', tarfile, '.'], cwd=srcdir) + return tarfile + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/cmd-fetch b/src/cmd-fetch index 0975843ff8..193f55a82a 100755 --- a/src/cmd-fetch +++ b/src/cmd-fetch @@ -114,7 +114,7 @@ fi prepare_compose_overlays ${IGNORE_COSA_OVERRIDES_ARG} # shellcheck disable=SC2086 -runcompose --download-only ${args} +runcompose_tree --download-only ${args} # This stamp file signifies we successfully fetched once; it's # validated in cmd-build. touch "${fetch_stamp}" diff --git a/src/cmdlib.sh b/src/cmdlib.sh index b0cad72a17..9a716a663a 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -369,26 +369,42 @@ EOF # such as `--repo` (which is auto-derived from the builddir) and # `--unified-core` that we always want. Also dispatches to supermin if # we're running without support for nested containerization. -runcompose() { +runcompose_tree() { local tmp_overridesdir=${TMPDIR}/override - if [ -f "${tmp_overridesdir}/local-overrides.json" ]; then # we need our overrides to be at the end of the list set - "$@" --ex-lockfile="${tmp_overridesdir}/local-overrides.json" fi + impl_rpmostree_compose tree --unified-core "${manifest}" "$@" + if has_privileges; then + sudo chown -R -h "${USER}":"${USER}" "${tmprepo}" + fi +} + +runcompose_extensions() { + local outputdir=$1; shift + impl_rpmostree_compose extensions "$@" --output-dir "$outputdir" + if has_privileges; then + sudo chown -R -h "${USER}":"${USER}" "${outputdir}" + fi +} + +impl_rpmostree_compose() { + local cmd=$1; shift + local workdir=${workdir:-$(pwd)} + local repo=${tmprepo:-${workdir}/tmp/repo} rm -f "${changed_stamp}" # shellcheck disable=SC2086 - set - ${COSA_RPMOSTREE_GDB:-} rpm-ostree compose tree --repo="${tmprepo}" \ - --cachedir="${workdir}"/cache --touch-if-changed "${changed_stamp}" \ - --unified-core "${manifest}" ${COSA_RPMOSTREE_ARGS:-} "$@" + set - ${COSA_RPMOSTREE_GDB:-} rpm-ostree compose "${cmd}" --repo="${repo}" \ + --touch-if-changed "${changed_stamp}" --cachedir="${workdir}"/cache \ + ${COSA_RPMOSTREE_ARGS:-} "$@" echo "Running: $*" # this is the heart of the privs vs no privs dual path if has_privileges; then sudo -E "$@" - sudo chown -R -h "${USER}":"${USER}" "${tmprepo}" else # "cache2" has an explicit label so we can find it in qemu easily if [ ! -f "${workdir}"/cache/cache2.qcow2 ]; then diff --git a/src/cosalib/cmdlib.py b/src/cosalib/cmdlib.py index 7eaee371f3..bb32573309 100644 --- a/src/cosalib/cmdlib.py +++ b/src/cosalib/cmdlib.py @@ -33,6 +33,8 @@ retry_if_exception_type(IncompleteReadError) | retry_if_exception_type(ReadTimeoutError)) +THISDIR = os.path.dirname(os.path.abspath(__file__)) + def retry_callback(retry_state): print(f"Retrying after {retry_state.outcome.exception()}") @@ -309,3 +311,12 @@ def image_info(image): return out except Exception as e: raise Exception(f"failed to inspect {image} with qemu", e) + + +# Hackily run some bash code from cmdlib.sh helpers. +def cmdlib_sh(script): + subprocess.check_call(['bash', '-c', f''' + set -euo pipefail + source {THISDIR}/../cmdlib.sh + {script} + ''']) diff --git a/src/download-extensions b/src/download-extensions deleted file mode 100755 index 74d5710806..0000000000 --- a/src/download-extensions +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# RPMs as operating system extensions, distinct from the base ostree commit/image -# https://github.com/openshift/enhancements/blob/master/enhancements/rhcos/extensions.md - -import os -import sys -import yaml -from cosalib import cmdlib - -destdir = sys.argv[1] -tmpdir = 'tmp' -# yum wants this to be absolute -configdir = os.path.abspath('src/config') -extsrcpath = f'{configdir}/extensions.yaml' -extjson = f'{tmpdir}/extensions.json' -basearch = cmdlib.get_basearch() - -with open(extsrcpath) as f: - extensions = yaml.safe_load(f) - -# The "v2" format here is that there's an extensions/ directory, with subdirectories -# for each extension - except you should ignore "repodata/". -edestdir = f'{destdir}/extensions' -os.mkdir(edestdir) - -# Stuff that's not part of the extension -dependenciesdir = f'{edestdir}/dependencies' -os.mkdir(dependenciesdir) - - -# Downloads packages from specified repos -def yumdownload(destdir, pkgs): - # FIXME eventually use rpm-ostree for this - # shellcheck disable=SC2068 - args = ['yum', f'--setopt=reposdir={configdir}', f'--arch={basearch}', 'download'] - args.extend(pkgs) - cmdlib.run_verbose(args, cwd=destdir) - - -# Reuseable function for setting up an extension -# Assumes it is running in "${destdir}/extensions" -# 1 = extension name -# 2 = package string/glob -# 3 = OPTIONAL: dependencies string/glob -def createext(extname, pkgs): - print(f"Creating extension {extname}") - extdir = f"{edestdir}/{extname}" - os.mkdir(extdir) - primary = pkgs[0] - yumdownload(extdir, [primary]) - - deps = pkgs[1:] - if len(deps) > 0: - print(f"Downloading dependencies for {extname}") - yumdownload(dependenciesdir, deps) - - -for (name, ext) in extensions['extensions'].items(): - pkgs = ext['packages'] - extarches = ext.get('architectures') - if extarches is not None and basearch not in extarches: - print(f"Skipping extension {name} for this architecture") - continue - createext(name, pkgs) - -# Create the yum/dnf repo -cmdlib.run_verbose(['createrepo_c', '--no-database', '.'], cwd=edestdir) diff --git a/src/schema/v1.json b/src/schema/v1.json index 0d8e6ec35f..758a052a7f 100644 --- a/src/schema/v1.json +++ b/src/schema/v1.json @@ -181,6 +181,7 @@ "ibmcloud", "images", "oscontainer", + "extensions", "parent-pkgdiff", "pkgdiff", "release-payload", @@ -519,6 +520,41 @@ "title":"Oscontainer", "$ref": "#/definitions/image" }, + "extensions": { + "$id":"#/properties/extensions", + "type":"object", + "title":"Extensions", + "required": [ + "path", + "sha256", + "rpm-ostree-state", + "manifest" + ], + "properties": { + "path": { + "$id": "#/artifact/Path", + "type":"string", + "title":"Path" + }, + "sha256": { + "$id": "#/artifact/sha256", + "type":"string", + "title":"SHA256" + }, + "rpm-ostree-state": { + "$id":"#/properties/extensions/items/properties/rpm-ostree-state", + "type":"string", + "title":"RpmOstreeState", + "default":"", + "minLength": 64 + }, + "manifest": { + "$id":"#/properties/extensions/items/properties/manifest", + "type":"object", + "title":"Manifest" + } + } + }, "ostree-commit": { "$id":"#/properties/ostree-commit", "type":"string",