From 0e5db59371116271a29fadad9084a0b89fe0b336 Mon Sep 17 00:00:00 2001 From: Laura Barcziova Date: Tue, 23 Jul 2024 14:08:30 +0200 Subject: [PATCH] Finish OpenScanHub integration prototype Fixes #2463 --- packit_service/worker/handlers/copr.py | 161 +++++++++++++++++- .../worker/helpers/build/copr_build.py | 42 ++++- tests/unit/test_scan.py | 56 +++++- 3 files changed, 242 insertions(+), 17 deletions(-) diff --git a/packit_service/worker/handlers/copr.py b/packit_service/worker/handlers/copr.py index 9377e2c7aa..09982a034c 100644 --- a/packit_service/worker/handlers/copr.py +++ b/packit_service/worker/handlers/copr.py @@ -1,7 +1,9 @@ # Copyright Contributors to the Packit project. # SPDX-License-Identifier: MIT +import json import logging +import shutil import tempfile from datetime import datetime, timezone from os.path import basename @@ -21,10 +23,13 @@ from packit_service.constants import ( COPR_API_SUCC_STATE, COPR_SRPM_CHROOT, + DOCS_URL, ) from packit_service.models import ( CoprBuildTargetModel, BuildStatus, + ProjectEventModelType, + SRPMBuildModel, ) from packit_service.service.urls import get_copr_build_info_url, get_srpm_build_info_url from packit_service.utils import ( @@ -358,7 +363,14 @@ def run(self): self.set_built_packages() self.build.set_status(BuildStatus.success) self.handle_testing_farm() - # self.handle_scan() + + try: + self.handle_scan() + except Exception as ex: + logger.debug( + f"Handling the scan raised an exception: {ex}. Skipping " + f"as this is only experimental functionality for now." + ) return TaskResults(success=True, details={}) @@ -486,24 +498,155 @@ def handle_testing_farm(self): ).apply_async() def handle_scan(self): + """ + If configured, try to find a job that can provide the base SRPM, + download both SRPM and base SRPM and trigger the scan in OpenScanHub. + """ if ( - self.build.target != "fedora-rawhide-x86_64" + self.db_project_event.type != ProjectEventModelType.pull_request + or self.build.target != "fedora-rawhide-x86_64" or not self.job_config.osh_diff_scan_after_copr_build ): return - logger.info("About to trigger scan in OpenScanHub.") + base_build_job = self.find_base_build_job() + if not base_build_job: + logger.debug("No base build job needed for diff scan found in the config.") + return + + logger.info("Preparing to trigger scan in OpenScanHub...") - # TODO handle the base build check and download and add it to the run_osh_build call + base_srpm_model = self.get_base_srpm_model(base_build_job) - directory = tempfile.mkdtemp() + if not base_srpm_model: + return srpm_model = self.build.get_srpm_build() - srpm_path = Path(directory).joinpath(basename(srpm_model.url)) - if not download_file(srpm_model.url, srpm_path): + directory = tempfile.mkdtemp() + paths = self.download_srpms(directory, base_srpm_model, srpm_model) + if not paths: return - self.copr_build_helper.api.run_osh_build(srpm_path=srpm_path) + build_dashboard_url = get_copr_build_info_url(self.build.id) + + output = self.copr_build_helper.api.run_osh_build( + srpm_path=paths[1], + base_srpm=paths[0], + comment=f"Submitted via Packit Service for {build_dashboard_url}.", + ) + + if not output: + logger.debug("Something went wrong, skipping the reporting.") + return + + logger.info(f"Scan submitted successfully, output: {output}") + + response = json.loads(output) + url = response.get("url") + + if not url: + logger.debug("It was not possible to get the URL from the response.") + return + + self.copr_build_helper._report( + state=BaseCommitStatus.success, + description=( + "Scan in OpenScanHub submitted succesfully. Check the URL for more details." + ), + url=url, + check_names=["osh-diff-scan:fedora-rawhide-x86_64"], + markdown_content=( + "This is an experimental feature. Once the scan finishes, you can see the " + "newly introduced defects in the `added.html` in `Logs`. " + "You can disable the scanning in your configuration by " + "setting `osh_diff_scan_after_copr_build` to `False`. For more information, " + f"see [docs]({DOCS_URL}/configuration#osh_diff_scan_after_copr_build)." + ), + ) + + shutil.rmtree(directory) + + def find_base_build_job(self) -> Optional[JobConfig]: + """ + Find the job in the config that can provide the base build for the scan + (with `commit` trigger and same branch configured as the target PR branch). + """ + base_build_job = None + + for job in self.package_config.get_job_views(): + if ( + job.type in (JobType.copr_build, JobType.build) + and job.trigger == JobConfigTriggerType.commit + and ( + ( + job.branch + and job.branch + == self.copr_build_helper.pull_request_object.target_branch + ) + or ( + not job.branch + and self.project.default_branch + == self.copr_build_helper.pull_request_object.target_branch + ) + ) + ): + base_build_job = job + break + + return base_build_job + + def get_base_srpm_model( + self, base_build_job: JobConfig + ) -> Optional[SRPMBuildModel]: + """ + Get the SRPM build model of the latest successful Copr build + for the given job config. + """ + base_build_project_name = ( + self.copr_build_helper.job_project_for_commit_job_config(base_build_job) + ) + base_build_owner = self.copr_build_helper.job_owner_for_job_config( + base_build_job + ) + target_branch_commit = ( + self.copr_build_helper.pull_request_object.target_branch_head_commit + ) + + logger.debug( + f"Searching for base build for {target_branch_commit} commit " + f"in {base_build_owner}/{base_build_project_name} Copr project in our DB. " + ) + + builds = CoprBuildTargetModel.get_all_by( + commit_sha=target_branch_commit, + project_name=base_build_project_name, + owner=base_build_owner, + target="fedora-rawhide-x86_64", + status=BuildStatus.success, + ) + if not builds: + logger.debug("No matching base build found in our DB.") + return None + + return list(builds)[0].get_srpm_build() + + @staticmethod + def download_srpms( + directory: str, + base_srpm_model: SRPMBuildModel, + srpm_model: SRPMBuildModel, + ) -> Optional[tuple[Path, Path]]: + base_srpm_path = Path(directory).joinpath(basename(base_srpm_model.url)) + if not download_file(base_srpm_model.url, base_srpm_path): + logger.info( + f"Downloading of base SRPM {base_srpm_model.url} was not successful." + ) + return None + + srpm_path = Path(directory).joinpath(basename(srpm_model.url)) + if not download_file(srpm_model.url, srpm_path): + logger.info(f"Downloading of SRPM {srpm_model.url} was not successful.") + return None - # TODO reporting and cleanup + return base_srpm_path, srpm_path diff --git a/packit_service/worker/helpers/build/copr_build.py b/packit_service/worker/helpers/build/copr_build.py index caef584bd6..80c5c4b9b5 100644 --- a/packit_service/worker/helpers/build/copr_build.py +++ b/packit_service/worker/helpers/build/copr_build.py @@ -162,7 +162,7 @@ def normalise_copr_project_name(copr_project_name: str) -> str: @property def job_project(self) -> Optional[str]: """ - The job definition from the config file. + The project definition from the config file. """ if self.job_build and self.job_build.project: return self.job_build.project @@ -174,6 +174,36 @@ def job_project(self) -> Optional[str]: return self.default_project_name + def job_project_for_commit_job_config(self, job_config) -> Optional[str]: + """ + Get the Copr project name for the specified job config with commit trigger. + """ + if job_config.project: + return job_config.project + + service_hostname = parse_git_repo(self.project.service.instance_url).hostname + service_prefix = ( + "" if isinstance(self.project, GithubProject) else f"{service_hostname}-" + ) + + namespace = self.project.namespace.replace("/", "-") + + ref_identifier = job_config.branch or self.project.default_branch + + configured_identifier = ( + f"-{job_config.identifier}" if job_config.identifier else "" + ) + + if self.job_config.package: + configured_identifier = "" + + copr_project_name = ( + f"{service_prefix}{namespace}-{self.project.repo}-{ref_identifier}" + f"{configured_identifier}" + ) + + return self.normalise_copr_project_name(copr_project_name) + @property def job_owner(self) -> Optional[str]: """ @@ -188,6 +218,16 @@ def job_owner(self) -> Optional[str]: return test_job.owner return self.api.copr_helper.copr_client.config.get("username") + def job_owner_for_job_config(self, job_config: JobConfig) -> Optional[str]: + """ + Owner used for the copr build for the specified config + -- search the config or use the copr's config. + """ + if job_config.owner: + return job_config.owner + + return self.api.copr_helper.copr_client.config.get("username") + @property def preserve_project(self) -> Optional[bool]: """ diff --git a/tests/unit/test_scan.py b/tests/unit/test_scan.py index 5a65d461c6..89d0589f76 100644 --- a/tests/unit/test_scan.py +++ b/tests/unit/test_scan.py @@ -2,31 +2,73 @@ # SPDX-License-Identifier: MIT from packit.api import PackitAPI -from packit_service.models import CoprBuildTargetModel +from packit.config import JobType, JobConfigTriggerType +from packit_service.config import ServiceConfig +from packit_service.models import ( + CoprBuildTargetModel, + ProjectEventModelType, + BuildStatus, +) from packit_service.worker.handlers import CoprBuildEndHandler from packit_service.worker.handlers import copr from packit_service.worker.events import AbstractCoprBuildEvent from flexmock import flexmock +from packit_service.worker.helpers.build import CoprBuildJobHelper + def test_handle_scan(): srpm_mock = flexmock(url="https://some-url/my-srpm.src.rpm") flexmock(CoprBuildTargetModel).should_receive("get_by_build_id").and_return( flexmock( + id=1, get_srpm_build=lambda: srpm_mock, target="fedora-rawhide-x86_64", - get_project_event_model=lambda: None, + get_project_event_model=lambda: flexmock( + type=ProjectEventModelType.pull_request, + get_project_event_object=lambda: flexmock(), + ), ) ) flexmock(AbstractCoprBuildEvent).should_receive("from_event_dict").and_return( - flexmock(chroot="fedora-rawhide-x86_64", build_id="123") + flexmock(chroot="fedora-rawhide-x86_64", build_id="123", pr_id=12) + ) + + flexmock(copr).should_receive("download_file").twice().and_return(True) + project_mock = flexmock( + get_pr=lambda pr_id: flexmock( + target_branch="main", target_branch_head_commit="abcdef" + ) + ) + flexmock(ServiceConfig).should_receive("get_project").and_return(project_mock) + base_copr_model = flexmock(get_srpm_build=lambda: flexmock(url="base-srpm-url")) + + flexmock(CoprBuildTargetModel).should_receive("get_all_by").with_args( + commit_sha="abcdef", + project_name="commit-project", + owner="user-123", + target="fedora-rawhide-x86_64", + status=BuildStatus.success, + ).and_return([base_copr_model]) + + flexmock(PackitAPI).should_receive("run_osh_build").once().and_return( + '{"url": "scan-url"}' ) - flexmock(copr).should_receive("download_file").once().and_return(True) - flexmock(PackitAPI).should_receive("run_osh_build").once() + flexmock(CoprBuildJobHelper).should_receive("_report") CoprBuildEndHandler( - package_config=flexmock(), + package_config=flexmock( + get_job_views=lambda: [ + flexmock( + type=JobType.copr_build, + trigger=JobConfigTriggerType.commit, + branch="main", + project="commit-project", + owner="user-123", + ) + ] + ), job_config=flexmock(osh_diff_scan_after_copr_build=True), - event={}, + event={"pr_id": 12, "project_url": "https://some-project"}, ).handle_scan()