feat(reposcan): generate system profile for each RHEL release

RHINENG-15335

Dockerfile

@@ -7,7 +7,7 @@ RUN curl -o /etc/yum.repos.d/postgresql.repo \
 RUN microdnf module enable nginx:1.20 || :
 RUN microdnf module disable postgresql || :
 RUN microdnf install -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \
-        python312 python3.12-pip python3-rpm which nginx rpm-devel git-core shadow-utils diffutils systemd libicu postgresql go-toolset \
+        python312 python3.12-pip python3-rpm python3-dnf which nginx rpm-devel git-core shadow-utils diffutils systemd libicu postgresql go-toolset \
         $VAR_RPMS && \
         ln -s /usr/lib64/python3.6/site-packages/rpm /usr/lib64/python3.12/site-packages/rpm && \
     microdnf clean all
@@ -60,3 +60,5 @@ ADD /database                   /vmaas/database
 ADD /vmaas/webapp               /vmaas/vmaas/webapp
 ADD /vmaas/reposcan             /vmaas/vmaas/reposcan
 ADD /vmaas/common               /vmaas/vmaas/common
+
+ADD /vmaas/reposcan/redhatrelease/gen_package_profile.py /usr/local/bin

vmaas/reposcan/database/release_store.py

@@ -1,6 +1,8 @@
 """
 Module containing class for fetching/importing RHEL release metadata from/into database.
 """
+from psycopg2.extras import Json
+
 from vmaas.common.logging_utils import get_logger
 from vmaas.reposcan.database.database_handler import DatabaseHandler
 from vmaas.reposcan.mnm import RELEASE_FAILED_IMPORT
@@ -16,36 +18,57 @@ def __init__(self) -> None:
         self.logger = get_logger(__name__)
         self.conn = DatabaseHandler.get_connection()
 
+    def get_repositories(self, releases: list[str]) -> dict[str, list[tuple[str, str, str, str, str]]]:
+        """
+        Get list of repository URLs for each releasever.
+        (If the repository URL is available)
+        """
+        repositories: dict[str, list[tuple[str, str, str, str, str]]] = {}
+        cur = self.conn.cursor()
+        cur.execute("""SELECT r.releasever, cs.label, r.url, c.ca_cert, c.cert, c.key
+                       FROM content_set cs JOIN
+                            repo r ON cs.id = r.content_set_id JOIN
+                            arch a ON r.basearch_id = a.id JOIN
+                            certificate c ON r.certificate_id = c.id
+                       WHERE (cs.label SIMILAR TO 'rhel-_{1,2}-for-x86_64-(baseos|appstream)-rpms' OR
+                              cs.label SIMILAR TO 'rhel-_-server-rpms') AND
+                             a.name = 'x86_64' AND
+                             r.releasever = ANY(%s)
+                       ORDER BY r.releasever, cs.label, r.url""", (releases,))
+        for releasever, label, url, ca_cert, cert, key in cur.fetchall():
+            repositories.setdefault(releasever, []).append((label, url, ca_cert, cert, key))
+        cur.close()
+        return repositories
+
     def _sync_releases(self, releases: list[Release]) -> None:
-        self.logger.info("Syncing %d operating system releases.", len(releases))
         cur = self.conn.cursor()
         exists_in_db = {}
         try:
-            cur.execute("SELECT name, major, minor, ga FROM operating_system")
-            for name, major, minor, ga_date in cur.fetchall():
-                exists_in_db[(name, major, minor)] = ga_date
+            cur.execute("SELECT name, major, minor, ga, system_profile FROM operating_system")
+            for name, major, minor, ga_date, system_profile in cur.fetchall():
+                exists_in_db[(name, major, minor)] = (ga_date, system_profile)
 
             to_insert, to_update, to_delete = [], [], []
             for release in releases:
                 if (release.os_name, release.major, release.minor) not in exists_in_db:
                     to_insert.append(release)
                 else:
-                    if release.ga_date != exists_in_db[(release.os_name, release.major, release.minor)]:
+                    ga_date, system_profile = exists_in_db[(release.os_name, release.major, release.minor)]
+                    if release.ga_date != ga_date or release.system_profile != system_profile:
                         to_update.append(release)
                     del exists_in_db[(release.os_name, release.major, release.minor)]
-            to_delete.extend([Release(name, major, minor, ga_date) for (name, major, minor), ga_date in exists_in_db.items()])
+            to_delete.extend([Release(name, major, minor, ga_date, system_profile)
+                              for (name, major, minor), (ga_date, system_profile) in exists_in_db.items()])
 
-            self.logger.debug("Releases to insert: %d", len(to_insert))
-            self.logger.debug("Releases to update: %d", len(to_update))
-            self.logger.debug("Releases to delete: %d", len(to_delete))
+            self.logger.info("Syncing %d operating system releases. (i=%d, u=%d, d=%d)", len(releases), len(to_insert), len(to_update), len(to_delete))
 
             for release in to_insert:
-                cur.execute("INSERT INTO operating_system (name, major, minor, ga) VALUES (%s, %s, %s, %s)",
-                            (release.os_name, release.major, release.minor, release.ga_date))
+                cur.execute("INSERT INTO operating_system (name, major, minor, ga, system_profile) VALUES (%s, %s, %s, %s, %s)",
+                            (release.os_name, release.major, release.minor, release.ga_date, Json(release.system_profile)))
 
             for release in to_update:
-                cur.execute("UPDATE operating_system SET ga = %s WHERE name = %s AND major = %s AND minor = %s",
-                            (release.ga_date, release.os_name, release.major, release.minor))
+                cur.execute("UPDATE operating_system SET ga = %s, system_profile = %s WHERE name = %s AND major = %s AND minor = %s",
+                            (release.ga_date, Json(release.system_profile), release.os_name, release.major, release.minor))
 
             for release in to_delete:
                 cur.execute("DELETE FROM operating_system WHERE name = %s AND major = %s AND minor = %s",

vmaas/reposcan/redhatrelease/gen_package_profile.py

@@ -0,0 +1,61 @@
+#!/usr/libexec/platform-python
+# mypy: disable-error-code="import-not-found"
+"""
+External script to generate system package profile using DNF.
+
+This script is executed using platform Python and not imported directly
+due to incompatibility between platform Python version of the libdnf (and others) - 3.6
+and Python version running the application - 3.12.
+
+This code might be included in the application after the running image is upgraded from UBI 8.
+"""
+import json
+import os
+import sys
+
+import dnf
+
+DNF_CACHEDIR = os.getenv("DNF_CACHEDIR", "")
+DNF_REPOS = json.loads(os.getenv("DNF_REPOS", "[]"))
+DNF_PLATFORM_ID = os.getenv("DNF_PLATFORM_ID", "")
+
+
+def _write_cert(label: str, cert_type: str, content: str) -> str:
+    cert_path = os.path.join(DNF_CACHEDIR, f"{label}-{cert_type}")
+    with open(cert_path, "w", encoding='utf8') as cert_file:
+        cert_file.write(content)
+    return cert_path
+
+
+def main() -> None:
+    if not all([DNF_CACHEDIR, DNF_REPOS, DNF_PLATFORM_ID]):
+        print("Some config ENV is not set!", file=sys.stderr)
+        sys.exit(1)
+
+    base = dnf.Base()
+    base.conf.cachedir = DNF_CACHEDIR
+    base.conf.installroot = DNF_CACHEDIR
+    base.conf.substitutions["arch"] = "x86_64"
+    base.conf.module_platform_id = DNF_PLATFORM_ID
+    for label, url, ca_cert, cert, key in DNF_REPOS:
+        ca_cert_path = _write_cert(label, "ca_cert", ca_cert)
+        cert_path = _write_cert(label, "cert", cert)
+        key_path = _write_cert(label, "key", key)
+        base.repos.add_new_repo(label,
+                                base.conf,
+                                baseurl=[url],
+                                sslcacert=ca_cert_path,
+                                sslclientcert=cert_path,
+                                sslclientkey=key_path)
+    base.fill_sack(load_system_repo=False, load_available_repos=True)
+    base.install_specs(["kernel", "@core", "@base"])
+    base.resolve()
+    system_profile = {"package_list": [f"{pkg.name}-{pkg.epoch}:{pkg.version}-{pkg.release}.{pkg.arch}"
+                                       for pkg in base.transaction.install_set]}
+    base.close()
+
+    print(json.dumps(system_profile))
+
+
+if __name__ == "__main__":
+    main()

vmaas/reposcan/redhatrelease/modeling.py

@@ -12,3 +12,4 @@ class Release:
     major: int
     minor: int
     ga_date: date
+    system_profile: dict[str, str | list[str]]

vmaas/reposcan/redhatrelease/release_controller.py

@@ -2,11 +2,15 @@
 Module containing class for syncing RHEL release metadata into database.
 """
 from datetime import date
+import json
+import subprocess
+import tempfile
 
 from vmaas.common.logging_utils import get_logger
 from vmaas.reposcan.database.release_store import ReleaseStore
 from vmaas.reposcan.redhatrelease.modeling import Release
 
+GEN_PACKAGES_SCRIPT = "/usr/local/bin/gen_package_profile.py"
 OS_DB_NAME = "RHEL"
 
 
@@ -17,18 +21,52 @@ def __init__(self) -> None:
         self.logger = get_logger(__name__)
         self.release_store = ReleaseStore()
 
+    def _get_dnf_package_list(self, release: Release, repositories: dict[str, list[tuple[str, str, str, str, str]]]) -> list[str]:
+        with tempfile.TemporaryDirectory(prefix=f"dnf-{release.major}.{release.minor}-") as tmpdirname:
+            dnf_env = {"DNF_CACHEDIR": tmpdirname,
+                       "DNF_REPOS": json.dumps(repositories[f"{release.major}.{release.minor}"]),
+                       "DNF_PLATFORM_ID": f"platform:el{release.major}"}
+            proc = subprocess.run([GEN_PACKAGES_SCRIPT], env=dnf_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if proc.returncode != 0:
+            self.logger.error("Executing package generation script failed: %s", proc.stderr)
+            raise ValueError("Unable to get valid package list")
+
+        # Extract package list from external script output
+        package_list: list[str] = json.loads(proc.stdout)["package_list"]
+        return package_list
+
+    def _generate_system_profile(self, release: Release, latest_releases: dict[int, int],
+                                 repositories: dict[str, list[tuple[str, str, str, str, str]]]) -> None:
+        self.logger.info("Generating system profile for release %d.%d", release.major, release.minor)
+        release.system_profile["package_list"] = sorted(self._get_dnf_package_list(release, repositories))
+        release.system_profile["repository_list"] = sorted([label for (label, _, _, _, _) in repositories[f"{release.major}.{release.minor}"]])
+        release.system_profile["basearch"] = "x86_64"
+        release.system_profile["releasever"] = f"{release.major}.{latest_releases[release.major]}"
+
     def _prepare_data(self, releases: dict[str, str]) -> list[Release]:
+        repositories = self.release_store.get_repositories(list(releases))
+        latest_releases: dict[int, int] = {}
         release_list = []
-        for release, ga_date in releases.items():
-            major, minor = release.split(".", 1)
+        for release_s, ga_date_s in releases.items():
+            ga_date = date.fromisoformat(ga_date_s)
+            # Filter out release versions not out yet, or those without repositories
+            if date.today() < ga_date or release_s not in repositories:
+                continue
+            major_s, minor_s = release_s.split(".", 1)
+            major, minor = int(major_s), int(minor_s)
+            if major not in latest_releases or minor > latest_releases[major]:
+                latest_releases[major] = minor
             release_list.append(
                 Release(
                     OS_DB_NAME,
-                    int(major),
-                    int(minor),
-                    date.fromisoformat(ga_date)
+                    major,
+                    minor,
+                    ga_date,
+                    {}
                 )
             )
+        for release in release_list:
+            self._generate_system_profile(release, latest_releases, repositories)
         return release_list
 
     def store(self, releases: dict[str, str]) -> None: