ocp4: clean up sources at the end of the artcd pipeline

pyartcd/pyartcd/pipelines/ocp4.py

@@ -83,7 +83,6 @@ def __init__(self, runtime: Runtime, version: str, assembly: str, data_path: str
         self.comment_on_pr = comment_on_pr
         self.copy_links = copy_links
 
-        self._doozer_working = os.path.abspath(f'{self.runtime.working_dir / "doozer_working"}')
         self.data_path = data_path
         self.data_gitref = data_gitref
         self.success_nvrs = []
@@ -98,7 +97,7 @@ def __init__(self, runtime: Runtime, version: str, assembly: str, data_path: str
         self._doozer_base_command = [
             'doozer',
             f'--assembly={assembly}',
-            f'--working-dir={self._doozer_working}',
+            f'--working-dir={self.runtime.doozer_working}',
             f'--data-path={data_path}',
             group_param
         ]
@@ -111,7 +110,7 @@ async def _check_assembly(self):
         If assembly != 'stream' and assemblies not enabled for <version>, raise an error
         """
 
-        shutil.rmtree(self._doozer_working, ignore_errors=True)
+        shutil.rmtree(self.runtime.doozer_working, ignore_errors=True)
         cmd = self._doozer_base_command.copy()
         cmd.extend(['config:read-group', '--default=False', 'assemblies.enabled'])
         _, out, err = await exectools.cmd_gather_async(cmd)
@@ -132,7 +131,7 @@ async def _initialize_version(self):
         """
 
         # version.branch
-        shutil.rmtree(self._doozer_working, ignore_errors=True)
+        shutil.rmtree(self.runtime.doozer_working, ignore_errors=True)
         cmd = self._doozer_base_command.copy()
         cmd.extend(['config:read-group', 'branch'])
         _, out, _ = await exectools.cmd_gather_async(cmd)
@@ -153,7 +152,7 @@ async def _initialize_build_plan(self):
         """
 
         # build_plan.active_image_count
-        shutil.rmtree(self._doozer_working, ignore_errors=True)
+        shutil.rmtree(self.runtime.doozer_working, ignore_errors=True)
         cmd = self._doozer_base_command.copy()
         cmd.append('images:list')
         _, out, _ = await exectools.cmd_gather_async(cmd)  # Last line looks like this: "219 images"
@@ -222,7 +221,7 @@ async def _is_build_permitted(self):
         return await util.is_build_permitted(
             version=self.version.stream,
             data_path=self.data_path,
-            doozer_working=self._doozer_working,
+            doozer_working=self.runtime.doozer_working,
             doozer_data_gitref=self.data_gitref)
 
     async def _initialize(self):
@@ -360,7 +359,7 @@ async def _rebase_and_build_rpms(self):
             self._handle_rpm_build_failures()
 
         try:
-            with open(f'{self._doozer_working}/record.log', 'r') as file:
+            with open(f'{self.runtime.doozer_working}/record.log', 'r') as file:
                 record_log: dict = record_util.parse_record_log(file)
 
             success_map = record_util.get_successful_rpms(record_log, full_record=True)
@@ -375,7 +374,7 @@ async def _rebase_and_build_rpms(self):
             self.runtime.logger.error(f"Failed to get successfully build RPM NVRs: {e}")
 
     def _handle_rpm_build_failures(self):
-        with open(f'{self._doozer_working}/record.log', 'r') as file:
+        with open(f'{self.runtime.doozer_working}/record.log', 'r') as file:
             record_log: dict = record_util.parse_record_log(file)
 
         failed_map = record_util.get_failed_rpms(record_log)
@@ -397,7 +396,7 @@ async def _is_compose_build_permitted(self) -> bool:
         automation_state: str = await util.get_freeze_automation(
             version=self.version.stream,
             doozer_data_path=self.data_path,
-            doozer_working=self._doozer_working,
+            doozer_working=self.runtime.doozer_working,
             doozer_data_gitref=self.data_gitref
         )
         self.runtime.logger.info('Automation freeze for %s: %s', self.version.stream, automation_state)
@@ -434,7 +433,7 @@ async def _build_compose(self):
                 stream=self.version.stream,
                 release=self.version.release,
                 assembly=self.assembly,
-                doozer_working=self._doozer_working,
+                doozer_working=self.runtime.doozer_working,
                 data_path=self.data_path,
                 data_gitref=self.data_gitref,
                 dry_run=self.runtime.dry_run,
@@ -489,7 +488,7 @@ async def _rebase_images(self):
 
         except ChildProcessError:
             # Get a list of images that failed to rebase
-            with open(f'{self._doozer_working}/state.yaml') as state_yaml:
+            with open(f'{self.runtime.doozer_working}/state.yaml') as state_yaml:
                 state = yaml.safe_load(state_yaml)
             failed_images = list(
                 dict(
@@ -511,19 +510,19 @@ async def _rebase_images(self):
 
         util.notify_dockerfile_reconciliations(
             version=self.version.stream,
-            doozer_working=self._doozer_working,
+            doozer_working=self.runtime.doozer_working,
             mail_client=self._mail_client
         )
 
         # TODO: if a non-required rebase fails, notify ART and the image owners
         util.notify_bz_info_missing(
             version=self.version.stream,
-            doozer_working=self._doozer_working,
+            doozer_working=self.runtime.doozer_working,
             mail_client=self._mail_client
         )
 
     def _handle_image_build_failures(self):
-        with open(f'{self._doozer_working}/record.log', 'r') as file:
+        with open(f'{self.runtime.doozer_working}/record.log', 'r') as file:
             record_log: dict = record_util.parse_record_log(file)
 
         failed_map = record_util.get_failed_builds(record_log, full_record=True)
@@ -554,7 +553,7 @@ def _handle_image_build_failures(self):
         else:
             util.mail_build_failure_owners(
                 failed_builds=failed_map,
-                doozer_working=self._doozer_working,
+                doozer_working=self.runtime.doozer_working,
                 mail_client=self._mail_client,
                 default_owner=self.mail_list_failure
             )
@@ -599,7 +598,7 @@ async def _build_images(self):
         # break CI builds for most upstream components if we don't catch it before we push. So we use apiserver as
         # bellweather to make sure that the current builder image is good enough. We can still break CI (e.g. pushing a
         # bad ruby-25 image along with this push, but it will not be a catastrophic event like breaking the apiserver.
-        with open(f'{self._doozer_working}/record.log', 'r') as file:
+        with open(f'{self.runtime.doozer_working}/record.log', 'r') as file:
             record_log: dict = record_util.parse_record_log(file)
 
         success_map = record_util.get_successful_builds(record_log, full_record=True)
@@ -643,7 +642,7 @@ async def _sync_images(self):
 
         self.runtime.logger.info('Syncing built images')
 
-        with open(f'{self._doozer_working}/record.log', 'r') as file:
+        with open(f'{self.runtime.doozer_working}/record.log', 'r') as file:
             record_log: dict = record_util.parse_record_log(file)
 
         records = record_log.get('build', [])
@@ -764,7 +763,7 @@ def _report_success(self):
         if self.runtime.dry_run or (not self.build_plan.build_rpms and not self.build_plan.build_images):
             record_log = {}  # Nothing was actually built
         else:
-            with open(f'{self._doozer_working}/record.log', 'r') as file:
+            with open(f'{self.runtime.doozer_working}/record.log', 'r') as file:
                 record_log: dict = record_util.parse_record_log(file)
         metrics = record_log.get('image_build_metrics', None)
 
@@ -850,6 +849,7 @@ async def run(self):
 
         # All good
         self._report_success()
+        await self.runtime.cleanup_sources('sources')
 
 
 @cli.command("ocp4",

pyartcd/pyartcd/pipelines/ocp4_konflux.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 import os
@@ -51,7 +52,6 @@ def __init__(self, runtime: Runtime, assembly: str, data_path: Optional[str], im
                  kubeconfig: Optional[str], skip_rebase: bool, arches: Tuple[str, ...], plr_template: str):
         self.runtime = runtime
         self.assembly = assembly
-        self._doozer_working = os.path.abspath(f'{self.runtime.working_dir / "doozer_working"}')
         self.version = version
         self.kubeconfig = kubeconfig
         self.arches = arches
@@ -65,7 +65,7 @@ def __init__(self, runtime: Runtime, assembly: str, data_path: Optional[str], im
         self._doozer_base_command = [
             'doozer',
             f'--assembly={assembly}',
-            f'--working-dir={self._doozer_working}',
+            f'--working-dir={self.runtime.doozer_working}',
             f'--data-path={data_path}',
             group_param
         ]
@@ -103,7 +103,7 @@ async def rebase(self, version: str, input_release: str):
             await exectools.cmd_assert_async(cmd)
 
         except ChildProcessError:
-            with open(f'{self._doozer_working}/state.yaml') as state_yaml:
+            with open(f'{self.runtime.doozer_working}/state.yaml') as state_yaml:
                 state = yaml.safe_load(state_yaml)
             failed_images = state['images:konflux:rebase'].get('failed-images', [])
             if not failed_images:
@@ -152,7 +152,7 @@ async def build(self):
 
     async def init_build_plan(self):
         # Get number of images in current group
-        shutil.rmtree(self._doozer_working, ignore_errors=True)
+        shutil.rmtree(self.runtime.doozer_working, ignore_errors=True)
         _, out, _ = await exectools.cmd_gather_async([*self._doozer_base_command.copy(), 'images:list'])
         # Last line looks like this: "219 images"
         self.build_plan.active_image_count = int(out.splitlines()[-1].split(' ')[0].strip())
@@ -200,6 +200,13 @@ async def initialize(self):
         if self.assembly.lower() == "test":
             jenkins.update_title(" [TEST]")
 
+    async def clean_up(self):
+        LOGGER.info('Cleaning up Doozer source dirs')
+        await asyncio.gather(*[
+            self.runtime.cleanup_sources('sources'),
+            self.runtime.cleanup_sources('konflux_build_sources'),
+        ])
+
     async def run(self):
         await self.initialize()
 
@@ -215,6 +222,8 @@ async def run(self):
         LOGGER.info(f"Building images for OCP {self.version} with release {input_release}")
         await self.build()
 
+        await self.clean_up()
+
 
 @cli.command("beta:ocp4-konflux", help="A pipeline to build images with Konflux for OCP 4")
 @click.option('--image-build-strategy', required=True,

pyartcd/pyartcd/runtime.py

@@ -1,11 +1,13 @@
 import logging
 import os
+from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, Optional
 
 import tomli
 
 from artcommonlib import runtime, model
+from artcommonlib.exectools import cmd_gather_async
 from pyartcd import jenkins, util, constants
 from pyartcd.jira import JIRAClient
 from pyartcd.mail import MailService
@@ -16,6 +18,7 @@ class Runtime:
     def __init__(self, config: Dict[str, Any], working_dir: Path, dry_run: bool):
         self.config = config
         self.working_dir = working_dir
+        self.doozer_working = os.path.abspath(f'{self.working_dir / "doozer_working"}')
         self.dry_run = dry_run
         self.logger = self.init_logger()
 
@@ -60,6 +63,23 @@ def new_slack_client(self, token: Optional[str] = None):
     def new_mail_client(self):
         return MailService.from_config(self.config)
 
+    async def cleanup_sources(self, source_dir_name: str):
+        """
+        Removes the git sources cloned by Doozer using rsync
+        """
+
+        source_path = f'{self.doozer_working}/{source_dir_name}'
+        self.logger.info('About to remove source dir %s...', source_path)
+        empty_to_overwrite_path = f'{self.doozer_working}/empty_to_overwrite/'
+
+        # Create the empty directory and delete the source dir with rsync --delete
+        Path(empty_to_overwrite_path).mkdir(parents=True, exist_ok=True)
+        cmd = ['rsync', '-a', '--delete', f'{empty_to_overwrite_path}', source_path]
+        self.logger.info('Executing rsync delete operation: %s', ' '.join(cmd))
+        start_time = datetime.now()
+        await cmd_gather_async(cmd)
+        self.logger.info('Deleted %s in %s seconds', source_path, (datetime.now() - start_time).seconds)
+
 
 class GroupRuntime(runtime.GroupRuntime):
 

pyartcd/tests/pipelines/test_ocp4.py

@@ -258,7 +258,7 @@ def tearDown(self) -> None:
     @patch("os.path.abspath", return_value='doozer_working')
     def default_ocp4_pipeline(*_) -> ocp4.Ocp4Pipeline:
         pipeline = ocp4.Ocp4Pipeline(
-            runtime=MagicMock(dry_run=False),
+            runtime=MagicMock(dry_run=False, doozer_working='doozer_working'),
             assembly='stream',
             version='4.13',
             data_path=constants.OCP_BUILD_DATA_URL,
@@ -575,7 +575,7 @@ class TestUpdateDistgit(unittest.IsolatedAsyncioTestCase):
     @patch("artcommonlib.exectools.cmd_assert_async")
     async def test_update_distgit(self, cmd_assert_mock: AsyncMock, bz_info_missing_mock, reconciliations_mock, *_):
         pipeline = ocp4.Ocp4Pipeline(
-            runtime=MagicMock(dry_run=False),
+            runtime=MagicMock(dry_run=False, doozer_working='doozer_working'),
             assembly='stream',
             version='4.13',
             data_path=constants.OCP_BUILD_DATA_URL,