Finish OpenScanHub integration prototype

Fixes packit#2463

packit_service/worker/handlers/copr.py

@@ -1,6 +1,7 @@
 # Copyright Contributors to the Packit project.
 # SPDX-License-Identifier: MIT
 
+import json
 import logging
 import tempfile
 from datetime import datetime, timezone
@@ -21,10 +22,13 @@
 from packit_service.constants import (
     COPR_API_SUCC_STATE,
     COPR_SRPM_CHROOT,
+    DOCS_URL,
 )
 from packit_service.models import (
     CoprBuildTargetModel,
     BuildStatus,
+    ProjectEventModelType,
+    SRPMBuildModel,
 )
 from packit_service.service.urls import get_copr_build_info_url, get_srpm_build_info_url
 from packit_service.utils import (
@@ -72,6 +76,7 @@
     GetCoprBuildJobHelperMixin,
     ConfigFromEventMixin,
 )
+from packit_service.worker.helpers.build import CoprBuildJobHelper
 from packit_service.worker.mixin import PackitAPIWithDownstreamMixin
 from packit_service.worker.reporting import BaseCommitStatus, DuplicateCheckMode
 from packit_service.worker.result import TaskResults
@@ -358,7 +363,22 @@ def run(self):
         self.set_built_packages()
         self.build.set_status(BuildStatus.success)
         self.handle_testing_farm()
-        # self.handle_scan()
+
+        if (
+            self.db_project_event.type == ProjectEventModelType.pull_request
+            and self.build.target == "fedora-rawhide-x86_64"
+            and self.job_config.osh_diff_scan_after_copr_build
+        ):
+            try:
+                ScanHelper(
+                    build_job_helper=self.copr_build_helper,
+                    build=self.build,
+                ).handle_scan()
+            except Exception as ex:
+                logger.debug(
+                    f"Handling the scan raised an exception: {ex}. Skipping "
+                    f"as this is only experimental functionality for now."
+                )
 
         return TaskResults(success=True, details={})
 
@@ -485,25 +505,159 @@ def handle_testing_farm(self):
                     },
                 ).apply_async()
 
+
+class ScanHelper:
+    def __init__(
+        self, copr_build_helper: CoprBuildJobHelper, build: CoprBuildTargetModel
+    ):
+        self.build = build
+        self.copr_build_helper = copr_build_helper
+
     def handle_scan(self):
-        if (
-            self.build.target != "fedora-rawhide-x86_64"
-            or not self.job_config.osh_diff_scan_after_copr_build
-        ):
+        """
+        Try to find a job that can provide the base SRPM,
+        download both SRPM and base SRPM and trigger the scan in OpenScanHub.
+        """
+        base_build_job = self.find_base_build_job()
+        if not base_build_job:
+            logger.debug("No base build job needed for diff scan found in the config.")
             return
 
-        logger.info("About to trigger scan in OpenScanHub.")
+        logger.info("Preparing to trigger scan in OpenScanHub...")
 
-        # TODO handle the base build check and download and add it to the run_osh_build call
+        base_srpm_model = self.get_base_srpm_model(base_build_job)
 
-        directory = tempfile.mkdtemp()
+        if not base_srpm_model:
+            return
 
         srpm_model = self.build.get_srpm_build()
-        srpm_path = Path(directory).joinpath(basename(srpm_model.url))
 
-        if not download_file(srpm_model.url, srpm_path):
-            return
+        with tempfile.TemporaryDirectory() as directory:
+            paths = self.download_srpms(directory, base_srpm_model, srpm_model)
+            if not paths:
+                return
+
+            build_dashboard_url = get_copr_build_info_url(self.build.id)
+
+            output = self.copr_build_helper.api.run_osh_build(
+                srpm_path=paths[1],
+                base_srpm=paths[0],
+                comment=f"Submitted via Packit Service for {build_dashboard_url}.",
+            )
 
-        self.copr_build_helper.api.run_osh_build(srpm_path=srpm_path)
+            if not output:
+                logger.debug("Something went wrong, skipping the reporting.")
+                return
 
-        # TODO reporting and cleanup
+            logger.info(f"Scan submitted successfully, output: {output}")
+
+            response = json.loads(output)
+            url = response.get("url")
+
+            if not url:
+                logger.debug("It was not possible to get the URL from the response.")
+                return
+
+            self.copr_build_helper._report(
+                state=BaseCommitStatus.success,
+                description=(
+                    "Scan in OpenScanHub submitted succesfully. Check the URL for more details."
+                ),
+                url=url,
+                check_names=["osh-diff-scan:fedora-rawhide-x86_64"],
+                markdown_content=(
+                    "This is an experimental feature. Once the scan finishes, you can see the "
+                    "newly introduced defects in the `added.html` in `Logs`. "
+                    "You can disable the scanning in your configuration by "
+                    "setting `osh_diff_scan_after_copr_build` to `False`. For more information, "
+                    f"see [docs]({DOCS_URL}/configuration#osh_diff_scan_after_copr_build)."
+                ),
+            )
+
+    def find_base_build_job(self) -> Optional[JobConfig]:
+        """
+        Find the job in the config that can provide the base build for the scan
+        (with `commit` trigger and same branch configured as the target PR branch).
+        """
+        base_build_job = None
+
+        for job in self.copr_build_helper.package_config.get_job_views():
+            if (
+                job.type in (JobType.copr_build, JobType.build)
+                and job.trigger == JobConfigTriggerType.commit
+                and (
+                    (
+                        job.branch
+                        and job.branch
+                        == self.copr_build_helper.pull_request_object.target_branch
+                    )
+                    or (
+                        not job.branch
+                        and self.copr_build_helper.project.default_branch
+                        == self.copr_build_helper.pull_request_object.target_branch
+                    )
+                )
+            ):
+                base_build_job = job
+                break
+
+        return base_build_job
+
+    def get_base_srpm_model(
+        self, base_build_job: JobConfig
+    ) -> Optional[SRPMBuildModel]:
+        """
+        Get the SRPM build model of the latest successful Copr build
+        for the given job config.
+        """
+        base_build_project_name = (
+            self.copr_build_helper.job_project_for_commit_job_config(base_build_job)
+        )
+        base_build_owner = self.copr_build_helper.job_owner_for_job_config(
+            base_build_job
+        )
+        target_branch_commit = (
+            self.copr_build_helper.pull_request_object.target_branch_head_commit
+        )
+
+        logger.debug(
+            f"Searching for base build for {target_branch_commit} commit "
+            f"in {base_build_owner}/{base_build_project_name} Copr project in our DB. "
+        )
+
+        builds = CoprBuildTargetModel.get_all_by(
+            commit_sha=target_branch_commit,
+            project_name=base_build_project_name,
+            owner=base_build_owner,
+            target="fedora-rawhide-x86_64",
+            status=BuildStatus.success,
+        )
+        if not builds:
+            logger.debug("No matching base build found in our DB.")
+            return None
+
+        return next(iter(builds)).get_srpm_build()
+
+    @staticmethod
+    def download_srpms(
+        directory: str,
+        base_srpm_model: SRPMBuildModel,
+        srpm_model: SRPMBuildModel,
+    ) -> Optional[tuple[Path, Path]]:
+
+        def download_srpm(srpm_model: SRPMBuildModel) -> Optional[Path]:
+            srpm_path = Path(directory).joinpath(basename(srpm_model.url))
+            if not download_file(srpm_model.url, srpm_path):
+                logger.info(f"Downloading of SRPM {srpm_model.url} was not successful.")
+                return None
+            return srpm_path
+
+        base_srpm_path = download_srpm(base_srpm_model)
+        if base_srpm_path is None:
+            return None
+
+        srpm_path = download_srpm(srpm_model)
+        if srpm_path is None:
+            return None
+
+        return base_srpm_path, srpm_path

packit_service/worker/helpers/build/copr_build.py

@@ -162,7 +162,7 @@ def normalise_copr_project_name(copr_project_name: str) -> str:
     @property
     def job_project(self) -> Optional[str]:
         """
-        The job definition from the config file.
+        The project definition from the config file.
         """
         if self.job_build and self.job_build.project:
             return self.job_build.project
@@ -174,6 +174,35 @@ def job_project(self) -> Optional[str]:
 
         return self.default_project_name
 
+    def job_project_for_commit_job_config(self, job_config) -> Optional[str]:
+        """
+        Get the Copr project name for the specified job config with commit trigger.
+        """
+        if job_config.project:
+            return job_config.project
+
+        service_hostname = parse_git_repo(self.project.service.instance_url).hostname
+        service_prefix = (
+            "" if isinstance(self.project, GithubProject) else f"{service_hostname}-"
+        )
+
+        namespace = self.project.namespace.replace("/", "-")
+
+        ref_identifier = job_config.branch or self.project.default_branch
+
+        configured_identifier = (
+            f"-{job_config.identifier}"
+            if job_config.identifier and not self.job_config.package
+            else ""
+        )
+
+        copr_project_name = (
+            f"{service_prefix}{namespace}-{self.project.repo}-{ref_identifier}"
+            f"{configured_identifier}"
+        )
+
+        return self.normalise_copr_project_name(copr_project_name)
+
     @property
     def job_owner(self) -> Optional[str]:
         """
@@ -188,6 +217,16 @@ def job_owner(self) -> Optional[str]:
                 return test_job.owner
         return self.api.copr_helper.copr_client.config.get("username")
 
+    def job_owner_for_job_config(self, job_config: JobConfig) -> Optional[str]:
+        """
+        Owner used for the copr build for the specified config
+         -- search the config or use the copr's config.
+        """
+        if job_config.owner:
+            return job_config.owner
+
+        return self.api.copr_helper.copr_client.config.get("username")
+
     @property
     def preserve_project(self) -> Optional[bool]:
         """

tests/unit/test_scan.py

@@ -1,32 +1,77 @@
 # Copyright Contributors to the Packit project.
 # SPDX-License-Identifier: MIT
 
+from flexmock import flexmock
+
 from packit.api import PackitAPI
-from packit_service.models import CoprBuildTargetModel
-from packit_service.worker.handlers import CoprBuildEndHandler
-from packit_service.worker.handlers import copr
+from packit.config import JobType, JobConfigTriggerType
+from packit_service.models import (
+    CoprBuildTargetModel,
+    ProjectEventModelType,
+    BuildStatus,
+)
 from packit_service.worker.events import AbstractCoprBuildEvent
-from flexmock import flexmock
+from packit_service.worker.handlers import copr
+from packit_service.worker.handlers.copr import ScanHelper
+from packit_service.worker.helpers.build import CoprBuildJobHelper
 
 
 def test_handle_scan():
     srpm_mock = flexmock(url="https://some-url/my-srpm.src.rpm")
-    flexmock(CoprBuildTargetModel).should_receive("get_by_build_id").and_return(
-        flexmock(
-            get_srpm_build=lambda: srpm_mock,
-            target="fedora-rawhide-x86_64",
-            get_project_event_model=lambda: None,
-        )
-    )
     flexmock(AbstractCoprBuildEvent).should_receive("from_event_dict").and_return(
-        flexmock(chroot="fedora-rawhide-x86_64", build_id="123")
+        flexmock(chroot="fedora-rawhide-x86_64", build_id="123", pr_id=12)
     )
+    flexmock(copr).should_receive("download_file").twice().and_return(True)
+
+    base_copr_model = flexmock(get_srpm_build=lambda: flexmock(url="base-srpm-url"))
 
-    flexmock(copr).should_receive("download_file").once().and_return(True)
-    flexmock(PackitAPI).should_receive("run_osh_build").once()
+    flexmock(CoprBuildTargetModel).should_receive("get_all_by").with_args(
+        commit_sha="abcdef",
+        project_name="commit-project",
+        owner="user-123",
+        target="fedora-rawhide-x86_64",
+        status=BuildStatus.success,
+    ).and_return([base_copr_model])
 
-    CoprBuildEndHandler(
-        package_config=flexmock(),
-        job_config=flexmock(osh_diff_scan_after_copr_build=True),
-        event={},
+    flexmock(PackitAPI).should_receive("run_osh_build").once().and_return(
+        '{"url": "scan-url"}'
+    )
+
+    flexmock(CoprBuildJobHelper).should_receive("_report")
+    package_config = flexmock(
+        get_job_views=lambda: [
+            flexmock(
+                type=JobType.copr_build,
+                trigger=JobConfigTriggerType.commit,
+                branch="main",
+                project="commit-project",
+                owner="user-123",
+            )
+        ]
+    )
+
+    project = flexmock(
+        get_pr=lambda pr_id: flexmock(
+            target_branch="main", target_branch_head_commit="abcdef"
+        )
+    )
+
+    ScanHelper(
+        build=flexmock(
+            id=1,
+            get_srpm_build=lambda: srpm_mock,
+            target="fedora-rawhide-x86_64",
+            get_project_event_model=lambda: flexmock(
+                type=ProjectEventModelType.pull_request,
+                get_project_event_object=lambda: flexmock(),
+            ),
+        ),
+        copr_build_helper=CoprBuildJobHelper(
+            service_config=flexmock(),
+            package_config=package_config,
+            project=project,
+            metadata=flexmock(pr_id=12),
+            db_project_event=flexmock(get_project_event_object=lambda: None),
+            job_config=flexmock(),
+        ),
     ).handle_scan()