Konflux: Add OLM bundle rebase/build support

artcommon/artcommonlib/konflux/konflux_db.py

@@ -156,7 +156,7 @@ async def get_latest_builds(self, names: typing.List[str], group: str,
                                 outcome: KonfluxBuildOutcome = KonfluxBuildOutcome.SUCCESS, assembly: str = 'stream',
                                 el_target: str = None, artifact_type: ArtifactType = None,
                                 engine: Engine = None, completed_before: datetime = None)\
-            -> typing.List[konflux_build_record.KonfluxBuildRecord]:
+            -> typing.List[konflux_build_record.KonfluxRecord]:
         """
         For a list of component names, run get_latest_build() in a concurrent pool executor.
         """
@@ -167,8 +167,9 @@ async def get_latest_builds(self, names: typing.List[str], group: str,
 
     async def get_latest_build(self, name: str, group: str, outcome: KonfluxBuildOutcome = KonfluxBuildOutcome.SUCCESS,
                                assembly: str = 'stream', el_target: str = None, artifact_type: ArtifactType = None,
-                               engine: Engine = None, completed_before: datetime = None, extra_patterns: dict = {}) \
-            -> typing.Optional[konflux_build_record.KonfluxBuildRecord]:
+                               engine: Engine = None, completed_before: datetime = None, extra_patterns: dict = {},
+                               strict: bool = False) \
+            -> typing.Optional[konflux_build_record.KonfluxRecord]:
         """
         Search for the latest Konflux build information in BigQuery.
 
@@ -181,6 +182,8 @@ async def get_latest_build(self, name: str, group: str, outcome: KonfluxBuildOut
         :param engine: 'brew' | 'konflux'
         :param completed_before: cut off timestamp for builds completion time
         :param extra_patterns: e.g. {'release': 'b45ea65'} will result in adding "AND release LIKE '%b45ea65%'" to the query
+        :param strict: If True, raise an exception if the build record is not found.
+        :return: The latest build record; None if the build record is not found.
         """
 
         # Table is partitioned by start_time. Perform an iterative search within 3-month windows, going back to 3 years
@@ -234,10 +237,42 @@ async def get_latest_build(self, name: str, group: str, outcome: KonfluxBuildOut
                 continue
 
         # If we got here, no builds have been found in the whole 36 months period
+        if strict:
+            raise ValueError(f"Build record for {name} not found.")
         self.logger.warning('No builds found for %s in %s with status %s in assembly %s and target %s',
                             name, group, outcome.value, assembly, el_target)
         return None
 
+    async def get_build_record(self, nvr: str, strict: bool = True) -> typing.Optional[KonfluxRecord]:
+        """ Get a build record by NVR.
+
+        :param nvr: The NVR of the build.
+        :param strict: If True, raise an exception if the build record is not found.
+        :return: The build record; None if the build record is not found.
+        """
+        base_clauses = [Column('nvr', String) == nvr]
+        started_before = datetime.now(tz=timezone.utc)
+        for window in range(12):
+            end_search = started_before - window * 3 * timedelta(days=30)
+            start_search = end_search - 3 * timedelta(days=30)
+
+            where_clauses = copy.copy(base_clauses)
+            where_clauses.extend([
+                Column('start_time', DateTime) >= start_search,
+                Column('start_time', DateTime) < end_search,
+            ])
+            results = await self.bq_client.select(where_clauses, limit=1)
+            try:
+                return self.from_result_row(next(results))
+            except StopIteration:
+                # No builds found in current window, shift to the earlier one
+                continue
+        # If we got here, no builds have been found in the whole 36 months period
+        if strict:
+            raise ValueError(f"Build record with NVR {nvr} not found.")
+        self.logger.warning('No builds found for NVR %s', nvr)
+        return None
+
     def from_result_row(self, row: Row) -> typing.Optional[KonfluxRecord]:
         """
         Given a google.cloud.bigquery.table.Row object, construct and return a KonfluxBuild object

artcommon/artcommonlib/util.py

@@ -1,3 +1,4 @@
+from functools import lru_cache
 import logging
 from typing import OrderedDict, Optional, Tuple, Iterable, List
 from datetime import datetime, timezone, timedelta, date
@@ -34,6 +35,7 @@ def remove_suffix(s: str, suffix: str) -> str:
         return s[:]
 
 
+@lru_cache(maxsize=512)
 def convert_remote_git_to_https(source_url: str):
     """
     Accepts a source git URL in ssh or https format and return it in a normalized
@@ -58,6 +60,7 @@ def convert_remote_git_to_https(source_url: str):
     return f'https://{server}/{org_repo}'
 
 
+@lru_cache(maxsize=512)
 def convert_remote_git_to_ssh(url):
     """
     Accepts a remote git URL and turns it into a git@

doozer/doozerlib/backend/build_repo.py

@@ -15,12 +15,12 @@ class BuildRepo:
 
     def __init__(self,
                  url: str,
-                 branch: str,
+                 branch: Optional[str],
                  local_dir: Union[str, Path],
                  logger: Optional[logging.Logger]) -> None:
         """ Initialize a BuildRepo object.
         :param url: The URL of the build source repository.
-        :param branch: The branch of the build source repository to clone.
+        :param branch: The branch of the build source repository to clone. None to not switch to any branch.
         :param local_dir: The local directory to clone the build source repository into.
         :param logger: A logger object to use for logging messages
         """
@@ -39,6 +39,7 @@ def https_url(self) -> str:
     @property
     def commit_hash(self) -> Optional[str]:
         """ Get the commit hash of the current commit in the build source repository.
+        Returns None if the branch has no commits yet.
         """
         return self._commit_hash
 
@@ -47,9 +48,10 @@ def exists(self) -> bool:
         """
         return self.local_dir.joinpath(".git").exists()
 
-    async def ensure_source(self, upcycle: bool = False):
+    async def ensure_source(self, upcycle: bool = False, strict: bool = False):
         """ Ensure that the build source repository is cloned into the local directory.
         :param upcycle: If True, the local directory will be deleted and recreated if it already exists.
+        :param strict: If True, raise an exception if the branch is not found in the build source repository.
         """
         local_dir = str(self.local_dir)
         needs_clone = True
@@ -59,33 +61,97 @@ async def ensure_source(self, upcycle: bool = False):
                 await exectools.to_thread(shutil.rmtree, local_dir)
             else:
                 self._logger.info("Reusing existing build source repository at %s", local_dir)
+                self._commit_hash = await self._get_commit_hash(local_dir)
                 needs_clone = False
         if needs_clone:
             self._logger.info("Cloning build source repository %s on branch %s into %s...", self.url, self.branch, self.local_dir)
-            await self.clone()
+            await self.clone(strict=strict)
 
-    async def clone(self):
+    async def clone(self, strict: bool = False):
         """ Clone the build source repository into the local directory.
+        :param strict: If True, raise an exception if the branch is not found in the build source repository.
         """
         local_dir = str(self.local_dir)
         self._logger.info("Cloning build source repository %s on branch %s into %s...", self.url, self.branch, local_dir)
+        await self.init()
+        await self.set_remote_url(self.url)
+        self._commit_hash = None
+        if self.branch is not None:
+            if await self.fetch(self.branch, strict=strict):
+                await self.switch(self.branch)
+            else:
+                self._logger.info("Branch %s not found in build source repository; creating a new branch instead", self.branch)
+                await self.switch(self.branch, orphan=True)
+
+    async def init(self):
+        """ Initialize the local directory as a git repository."""
+        local_dir = str(self.local_dir)
         await git_helper.run_git_async(["init", local_dir])
+
+    async def set_remote_url(self, url: Optional[str], remote_name: str = "origin"):
+        """ Set the URL of the remote in the build source repository.
+        :param url: The URL of the remote. None to use the default URL specified in the constructor.
+        :param remote_name: The name of the remote.
+        """
+        if url is None:
+            url = self.url
+        local_dir = str(self.local_dir)
         _, out, _ = await git_helper.gather_git_async(["-C", local_dir, "remote"], stderr=None)
-        if 'origin' not in out.strip().split():
-            await git_helper.run_git_async(["-C", local_dir, "remote", "add", "origin", self.url])
+        if remote_name not in out.strip().split():
+            await git_helper.run_git_async(["-C", local_dir, "remote", "add", remote_name, url])
         else:
-            await git_helper.run_git_async(["-C", local_dir, "remote", "set-url", "origin", self.url])
-        rc, _, err = await git_helper.gather_git_async(["-C", local_dir, "fetch", "--depth=1", "origin", self.branch], check=False)
+            await git_helper.run_git_async(["-C", local_dir, "remote", "set-url", remote_name, url])
+
+    async def fetch(self, refspec: str, depth: Optional[int] = 1, strict: bool = False):
+        """ Fetch a refspec from the build source repository.
+        :param refspec: The refspec to fetch.
+        :param depth: The depth of the fetch. None to fetch the entire history.
+        :param strict: If True, raise an exception if the refspec is not found in the remote.
+        :return: True if the fetch was successful; False otherwise
+        """
+        local_dir = str(self.local_dir)
+        fetch_options = []
+        if depth is not None:
+            fetch_options.append(f"--depth={depth}")
+        rc, _, err = await git_helper.gather_git_async(["-C", local_dir, "fetch"] + fetch_options + ["origin", refspec], check=False)
         if rc != 0:
-            if "fatal: couldn't find remote ref" in err:
-                self._logger.info("Branch %s not found in build source repository; creating a new branch instead", self.branch)
-                await git_helper.run_git_async(["-C", local_dir, "checkout", "--orphan", self.branch])
-            else:
-                raise ChildProcessError(f"Failed to fetch {self.branch} from {self.url}: {err}")
-        else:
-            await git_helper.run_git_async(["-C", local_dir, "checkout", "-B", self.branch, "-t", f"origin/{self.branch}"])
-            _, commit_hash, _ = await git_helper.gather_git_async(["-C", local_dir, "rev-parse", "HEAD"])
-            self._commit_hash = commit_hash.strip()
+            if not strict and "fatal: couldn't find remote ref" in err:
+                self._logger.warning("Failed to fetch %s from %s: %s", refspec, self.url, err)
+                return False
+            raise ChildProcessError(f"Failed to fetch {refspec} from {self.url}: {err}")
+        return True
+
+    async def switch(self, branch: str, detach: bool = False, orphan: bool = False):
+        """ Switch to a different branch in the build source repository.
+        :param branch: The branch to switch to.
+        """
+        local_dir = str(self.local_dir)
+        options = []
+        if detach:
+            options.append("--detach")
+        if orphan:
+            options.append("--orphan")
+        await git_helper.run_git_async(["-C", local_dir, "switch"] + options + [branch])
+        self.branch = branch
+        self._commit_hash = await self._get_commit_hash(local_dir)
+
+    async def delete_all_files(self):
+        """ Delete all files in the local directory.
+        """
+        await git_helper.run_git_async(["-C", str(self.local_dir), "rm", "-rf", "--ignore-unmatch", "."])
+
+    @staticmethod
+    async def _get_commit_hash(local_dir: str) -> Optional[str]:
+        """ Get the commit hash of the current commit in the build source repository.
+        :return: The commit hash of the current commit; None if the branch has no commits yet.
+        """
+        rc, out, err = await git_helper.gather_git_async(["-C", str(local_dir), "rev-parse", "HEAD"], check=False)
+        if rc != 0:
+            if "unknown revision or path not in the working tree" in err:
+                # This branch has no commits yet
+                return None
+            raise ChildProcessError(f"Failed to get commit hash: {err}")
+        return out.strip()
 
     async def commit(self, message: str, allow_empty: bool = False):
         """ Commit changes in the local directory to the build source repository."""
@@ -95,13 +161,20 @@ async def commit(self, message: str, allow_empty: bool = False):
         if allow_empty:
             commit_opts.append("--allow-empty")
         await git_helper.run_git_async(["-C", local_dir, "commit"] + commit_opts + ["-m", message])
-        _, out, _ = await git_helper.gather_git_async(["-C", local_dir, "rev-parse", "HEAD"])
-        self._commit_hash = out.strip()
+        self._commit_hash = await self._get_commit_hash(local_dir)
+
+    async def tag(self, tag: str):
+        """ Tag the current commit in the build source repository.
+
+        :param tag: The tag to apply to the current commit.
+        """
+        local_dir = str(self.local_dir)
+        await git_helper.run_git_async(["-C", local_dir, "tag", "-fam", tag, "--", tag])
 
     async def push(self):
         """ Push changes in the local directory to the build source repository."""
         local_dir = str(self.local_dir)
-        await git_helper.run_git_async(["-C", local_dir, "push", "origin", self.branch])
+        await git_helper.run_git_async(["-C", local_dir, "push", "--follow-tags", "origin", "HEAD"])
 
     @staticmethod
     async def from_local_dir(local_dir: Union[str, Path], logger: Optional[logging.Logger] = None):
@@ -116,7 +189,6 @@ async def from_local_dir(local_dir: Union[str, Path], logger: Optional[logging.L
         local_dir = str(local_dir)
         _, url, _ = await git_helper.gather_git_async(["-C", local_dir, "config", "--get", "remote.origin.url"])
         _, branch, _ = await git_helper.gather_git_async(["-C", local_dir, "rev-parse", "--abbrev-ref", "HEAD"])
-        _, commit_hash, _ = await git_helper.gather_git_async(["-C", local_dir, "rev-parse", "HEAD"])
         repo = BuildRepo(url.strip(), branch.strip(), local_dir, logger)
-        repo._commit_hash = commit_hash.strip()
+        repo._commit_hash = await BuildRepo._get_commit_hash(local_dir)
         return repo

doozer/doozerlib/backend/konflux_client.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+from functools import lru_cache
 from typing import Dict, List, Optional, Sequence, Union, cast
 
 import jinja2
@@ -37,6 +38,10 @@ def __init__(self, default_namespace: str, config: Configuration, dry_run: bool
         self.dry_run = dry_run
         self._logger = logger
 
+    def close(self):
+        """ Close the client. """
+        self.api_client.close()
+
     @staticmethod
     def from_kubeconfig(default_namespace: str, config_file: Optional[str], context: Optional[str], dry_run: bool = False, logger: logging.Logger = LOGGER) -> "KonfluxClient":
         """ Create a KonfluxClient from a kubeconfig file.
@@ -280,6 +285,16 @@ async def ensure_component(self, name: str, application: str, component_name: st
         component = self._new_component(name, application, component_name, image_repo, source_url, revision)
         return await self._create_or_replace(component)
 
+    @lru_cache
+    @staticmethod
+    def _get_pipelinerun_template():
+        """ Get the PipelineRun template."""
+        # TODO: In the future the PipelineRun template should be loaded from a remote git repo.
+        template_path = files("doozerlib").joinpath("backend", "konflux_image_build_pipelinerun.yaml")
+        template_content = template_path.read_text()
+        template = jinja2.Template(template_content, autoescape=True)
+        return template
+
     @staticmethod
     def _new_pipelinerun_for_image_build(generate_name: str, namespace: Optional[str], application_name: str, component_name: str,
                                          git_url: str, commit_sha: str, target_branch: str, output_image: str,
@@ -288,9 +303,7 @@ def _new_pipelinerun_for_image_build(generate_name: str, namespace: Optional[str
         if additional_tags is None:
             additional_tags = []
         https_url = art_util.convert_remote_git_to_https(git_url)
-        # TODO: In the future the PipelineRun template should be loaded from a remote git repo.
-        template_content = files("doozerlib").joinpath("backend").joinpath("konflux_image_build_pipelinerun.yaml").read_text()
-        template = jinja2.Template(template_content, autoescape=True)
+        template = KonfluxClient._get_pipelinerun_template()
         rendered = template.render({
             "source_url": https_url,
             "revision": commit_sha,
@@ -499,4 +512,4 @@ def _inner():
                         continue
                     raise
 
-        return await exectools.to_thread(_inner)
+        return await asyncio.to_thread(_inner)

doozer/doozerlib/backend/konflux_image_builder.py

@@ -78,7 +78,7 @@ async def build(self, metadata: ImageMetadata):
                 source = None
                 if metadata.has_source():
                     logger.info(f"Resolving source for {metadata.qualified_key}")
-                    source = cast(SourceResolution, await exectools.to_thread(metadata.runtime.source_resolver.resolve_source, metadata))
+                    source = cast(SourceResolution, await exectools.to_thread(metadata.runtime.source_resolver.resolve_source, metadata, no_clone=True))
                 else:
                     raise IOError(f"Image {metadata.qualified_key} doesn't have upstream source. This is no longer supported.")
                 dest_branch = "art-{group}-assembly-{assembly_name}-dgk-{distgit_key}".format_map({