diff --git a/alembic/versions/242a0cdb6d23_create_oshscanmodel.py b/alembic/versions/242a0cdb6d23_create_oshscanmodel.py new file mode 100644 index 000000000..004f58b5d --- /dev/null +++ b/alembic/versions/242a0cdb6d23_create_oshscanmodel.py @@ -0,0 +1,57 @@ +"""Create OSHScanModel + +Revision ID: 242a0cdb6d23 +Revises: f376be1907e1 +Create Date: 2024-10-18 08:10:35.335510 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "242a0cdb6d23" +down_revision = "f376be1907e1" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "scans", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("task_id", sa.Integer(), nullable=True), + sa.Column( + "status", + sa.Enum( + "pending", + "running", + "succeeded", + "canceled", + "failed", + name="oshscanstatus", + ), + nullable=True, + ), + sa.Column("url", sa.String(), nullable=True), + sa.Column("issues_added_url", sa.String(), nullable=True), + sa.Column("issues_fixed_url", sa.String(), nullable=True), + sa.Column("scan_results_url", sa.String(), nullable=True), + sa.Column("copr_build_target_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["copr_build_target_id"], + ["copr_build_targets.id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("copr_build_target_id"), + sa.UniqueConstraint("task_id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("scans") + # ### end Alembic commands ### diff --git a/packit_service/constants.py b/packit_service/constants.py index 67698def9..a8e68dc8e 100644 --- a/packit_service/constants.py +++ b/packit_service/constants.py @@ -302,3 +302,13 @@ def from_number(number: int): USAGE_PAST_YEAR_DATE_STR = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d") USAGE_DATE_IN_THE_PAST = USAGE_CURRENT_DATE.replace(year=USAGE_CURRENT_DATE.year - 100) USAGE_DATE_IN_THE_PAST_STR = USAGE_DATE_IN_THE_PAST.strftime("%Y-%m-%d") + +OPEN_SCAN_HUB_FEATURE_DESCRIPTION = ( + "This is an experimental feature. Once the scan finishes, you can see the " + "new findings in the `added.html` in `Logs`. \n\n" + ":warning: You can see the list of known issues and also provide your feedback" + " [here](https://github.com/packit/packit/discussions/2371). \n\n" + "You can disable the scanning in your configuration by " + "setting `osh_diff_scan_after_copr_build` to `false`. For more information, " + f"see [docs]({DOCS_URL}/configuration#osh_diff_scan_after_copr_build)." +) diff --git a/packit_service/models.py b/packit_service/models.py index 9303c7e41..bda98941c 100644 --- a/packit_service/models.py +++ b/packit_service/models.py @@ -1970,6 +1970,8 @@ class CoprBuildTargetModel(GroupAndTargetModelConnector, Base): "CoprBuildGroupModel", back_populates="copr_build_targets" ) + scan = relationship("OSHScanModel", back_populates="copr_build_target") + def set_built_packages(self, built_packages): with sa_session_transaction(commit=True) as session: self.built_packages = built_packages @@ -2195,6 +2197,13 @@ def __repr__(self): f"build_submitted_time={self.build_submitted_time})" ) + def add_scan(self, task_id: int) -> "OSHScanModel": + with sa_session_transaction(commit=True) as session: + scan = OSHScanModel.get_or_create(task_id) + scan.copr_build_target = self + session.add(scan) + return scan + class KojiBuildGroupModel(ProjectAndEventsConnector, GroupModel, Base): __tablename__ = "koji_build_groups" @@ -3824,6 +3833,74 @@ def get_by_koji_name(cls, koji_name: str) -> Optional["SidetagModel"]: return session.query(SidetagModel).filter_by(koji_name=koji_name).first() +class OSHScanStatus(str, enum.Enum): + """An enum of all possible build statuses""" + + pending = "pending" + running = "running" + succeeded = "succeeded" + failed = "failed" + canceled = "canceled" + + +class OSHScanModel(Base): + __tablename__ = "scans" + id = Column(Integer, primary_key=True) + task_id = Column(Integer, unique=True) # open scan hub id + status = Column(Enum(OSHScanStatus)) + url = Column(String) + issues_added_url = Column(String) + issues_fixed_url = Column(String) + scan_results_url = Column(String) + copr_build_target_id = Column( + Integer, ForeignKey("copr_build_targets.id"), unique=True + ) + copr_build_target = relationship( + "CoprBuildTargetModel", back_populates="scan", uselist=False + ) + + @classmethod + def get_or_create(cls, task_id: int) -> "OSHScanModel": + with sa_session_transaction(commit=True) as session: + scan = cls.get_by_task_id(task_id) + if not scan: + scan = cls() + scan.task_id = task_id + scan.status = OSHScanStatus.pending + session.add(scan) + return scan + + def set_status(self, status: OSHScanStatus) -> None: + with sa_session_transaction(commit=True) as session: + self.status = status + session.add(self) + + def set_url(self, url: str) -> None: + with sa_session_transaction(commit=True) as session: + self.url = url + session.add(self) + + def set_issues_added_url(self, issues_added_url: str) -> None: + with sa_session_transaction(commit=True) as session: + self.issues_added_url = issues_added_url + session.add(self) + + def set_issues_fixed_url(self, issues_fixed_url: str) -> None: + with sa_session_transaction(commit=True) as session: + self.issues_fixed_url = issues_fixed_url + session.add(self) + + def set_scan_results_url(self, scan_results_url: str) -> None: + with sa_session_transaction(commit=True) as session: + self.scan_results_url = scan_results_url + session.add(self) + + @classmethod + def get_by_task_id(cls, task_id: int) -> Optional["OSHScanModel"]: + with sa_session_transaction() as session: + return session.query(cls).filter_by(task_id=task_id).first() + + @cached(cache=TTLCache(maxsize=2048, ttl=(60 * 60 * 24))) def get_usage_data(datetime_from=None, datetime_to=None, top=10) -> dict: """ diff --git a/packit_service/worker/allowlist.py b/packit_service/worker/allowlist.py index ed489bd4b..05fa9ba13 100644 --- a/packit_service/worker/allowlist.py +++ b/packit_service/worker/allowlist.py @@ -41,6 +41,8 @@ ReleaseEvent, TestingFarmResultsEvent, CheckRerunEvent, + OpenScanHubTaskFinishedEvent, + OpenScanHubTaskStartedEvent, ) from packit_service.worker.events.gitlab import ReleaseGitlabEvent from packit_service.worker.events.koji import KojiBuildEvent, KojiBuildTagEvent @@ -485,6 +487,8 @@ def check_and_report( KojiBuildTagEvent, CheckRerunEvent, NewHotnessUpdateEvent, + OpenScanHubTaskFinishedEvent, + OpenScanHubTaskStartedEvent, ): self._check_unchecked_event, ( ReleaseEvent, diff --git a/packit_service/worker/checker/open_scan_hub.py b/packit_service/worker/checker/open_scan_hub.py new file mode 100644 index 000000000..33374bcdd --- /dev/null +++ b/packit_service/worker/checker/open_scan_hub.py @@ -0,0 +1,26 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import logging + +from packit.config import ( + aliases, +) +from packit_service.worker.checker.abstract import Checker + +logger = logging.getLogger(__name__) + + +class RawhideX86Target( + Checker, +): + def pre_check(self) -> bool: + branches = aliases.get_build_targets( + *self.job_config.targets, + ) + if "fedora-rawhide-x86_64" not in branches: + logger.debug( + "Skipping job configuration with no fedora-rawhide-x86_64 target." + ) + return False + return True diff --git a/packit_service/worker/events/__init__.py b/packit_service/worker/events/__init__.py index 654ed2cc1..068867a4f 100644 --- a/packit_service/worker/events/__init__.py +++ b/packit_service/worker/events/__init__.py @@ -50,6 +50,11 @@ from packit_service.worker.events.vm_image import VMImageBuildResultEvent +from packit_service.worker.events.open_scan_hub import ( + OpenScanHubTaskFinishedEvent, + OpenScanHubTaskStartedEvent, +) + __all__ = [ Event.__name__, EventData.__name__, @@ -86,4 +91,6 @@ ReleaseGitlabEvent.__name__, TagPushGitlabEvent.__name__, PullRequestFlagPagureEvent.__name__, + OpenScanHubTaskFinishedEvent.__name__, + OpenScanHubTaskStartedEvent.__name__, ] diff --git a/packit_service/worker/events/event.py b/packit_service/worker/events/event.py index 08c97af45..747000c5a 100644 --- a/packit_service/worker/events/event.py +++ b/packit_service/worker/events/event.py @@ -141,6 +141,24 @@ def from_event_dict(cls, event: dict): branches_override=branches_override, ) + def to_event(self) -> "Event": + """ + Create an instance of Event class from the data in this class. + """ + mod = __import__("packit_service.worker.events", fromlist=[self.event_type]) + kls = getattr(mod, self.event_type) + kwargs = copy.copy(self.event_dict) + # The following data should be reconstructed by the Event instance (when needed) + kwargs.pop("event_type", None) + kwargs.pop("event_id", None) + kwargs.pop("task_accepted_time", None) + kwargs.pop("build_targets_override", None) + kwargs.pop("tests_targets_override", None) + kwargs.pop("branches_override", None) + pr_id = kwargs.pop("_pr_id", None) + kwargs["pr_id"] = pr_id + return kls(**kwargs) + @property def project(self): if not self._project: @@ -367,12 +385,17 @@ def store_packages_config(self): self.db_project_event.set_packages_config(package_config_dict) def get_non_serializable_attributes(self): + """List here both non serializable attributes and attributes that + we want to skip from the dict because are not needed to re-create + the event. + """ return [ "_db_project_object", "_db_project_event", "_project", "_base_project", "_package_config", + "_package_config_searched", ] def get_dict(self, default_dict: Optional[Dict] = None) -> dict: @@ -618,10 +641,11 @@ def get_all_build_targets_by_status( statuses_to_filter_with=statuses_to_filter_with, ) - def get_dict(self, default_dict: Optional[Dict] = None) -> dict: - result = super().get_dict() - result.pop("_pull_request_object") - return result + def get_non_serializable_attributes(self): + return super().get_non_serializable_attributes() + [ + "fail_when_config_file_missing", + "_pull_request_object", + ] class AbstractResultEvent(AbstractForgeIndependentEvent): diff --git a/packit_service/worker/events/open_scan_hub.py b/packit_service/worker/events/open_scan_hub.py new file mode 100644 index 000000000..8d9876667 --- /dev/null +++ b/packit_service/worker/events/open_scan_hub.py @@ -0,0 +1,91 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import enum +from typing import Optional +from logging import getLogger + +from ogr.abstract import GitProject +from packit_service.config import ServiceConfig +from packit_service.worker.events.event import AbstractResultEvent +from packit_service.models import ( + AbstractProjectObjectDbType, + ProjectEventModel, + OSHScanModel, +) + +logger = getLogger(__name__) + + +class OpenScanHubTaskAbstractEvent(AbstractResultEvent): + + def __init__( + self, + task_id: int, + commit_sha: Optional[str] = None, + **kwargs, + ): + super().__init__(**kwargs) + + self.task_id = task_id + self.commit_sha = commit_sha + + self.scan = OSHScanModel.get_by_task_id(task_id) + self.build = None + if not self.scan: + logger.warning( + f"Scan with id {task_id} not found in the database." + " It should have been created when receiving the CoprBuildEndEvent" + " and should have been associated with the copr build." + ) + else: + self.build = self.scan.copr_build_target + project_event = self.build.get_project_event_model() + # commit_sha is needed by the StatusReporter + # and have to be serialized to be later found in the + # event metadata + self.commit_sha = ( + project_event.commit_sha if not self.commit_sha else self.commit_sha + ) + + def get_db_project_object(self) -> Optional[AbstractProjectObjectDbType]: + return self.build.get_project_event_object() + + def get_db_project_event(self) -> Optional[ProjectEventModel]: + return self.build.get_project_event_model() + + def get_project(self) -> GitProject: + return ServiceConfig.get_service_config().get_project( + self.db_project_object.project.project_url + ) + + def get_non_serializable_attributes(self): + # build and scan are not serializable + return super().get_non_serializable_attributes() + ["build", "scan"] + + +class OpenScanHubTaskFinishedEvent(OpenScanHubTaskAbstractEvent): + + class Status(str, enum.Enum): + success = "success" + cancel = "cancel" + interrupt = "interrupt" + fail = "fail" + + def __init__( + self, + status: Status, + issues_added_url: str, + issues_fixed_url: str, + scan_results_url: str, + **kwargs, + ): + super().__init__(**kwargs) + + self.status = status + self.issues_added_url = issues_added_url + self.issues_fixed_url = issues_fixed_url + self.scan_results_url = scan_results_url + + +class OpenScanHubTaskStartedEvent(OpenScanHubTaskAbstractEvent): ... diff --git a/packit_service/worker/handlers/__init__.py b/packit_service/worker/handlers/__init__.py index eb1a15cd2..92a1f30c7 100644 --- a/packit_service/worker/handlers/__init__.py +++ b/packit_service/worker/handlers/__init__.py @@ -38,6 +38,10 @@ VMImageBuildHandler, VMImageBuildResultHandler, ) +from packit_service.worker.handlers.open_scan_hub import ( + OpenScanHubTaskFinishedHandler, + OpenScanHubTaskStartedHandler, +) __all__ = [ Handler.__name__, @@ -55,4 +59,6 @@ GithubFasVerificationHandler.__name__, VMImageBuildHandler.__name__, VMImageBuildResultHandler.__name__, + OpenScanHubTaskFinishedHandler.__name__, + OpenScanHubTaskStartedHandler.__name__, ] diff --git a/packit_service/worker/handlers/abstract.py b/packit_service/worker/handlers/abstract.py index 126bb4300..d3f3915f2 100644 --- a/packit_service/worker/handlers/abstract.py +++ b/packit_service/worker/handlers/abstract.py @@ -191,6 +191,8 @@ class TaskName(str, enum.Enum): check_onboarded_projects = "task.check_onboarded_projects" koji_build_tag = "task.koji_build_tag" tag_into_sidetag = "task.tag_into_sidetag" + openscanhub_task_finished = "task.openscanhub_task_finished" + openscanhub_task_started = "task.openscanhub_task_started" class Handler(PackitAPIProtocol, Config): diff --git a/packit_service/worker/handlers/copr.py b/packit_service/worker/handlers/copr.py index 4c841fa70..4c800f3ad 100644 --- a/packit_service/worker/handlers/copr.py +++ b/packit_service/worker/handlers/copr.py @@ -1,14 +1,8 @@ # Copyright Contributors to the Packit project. # SPDX-License-Identifier: MIT -import json import logging -import re -import tempfile from datetime import datetime, timezone -from os import getenv -from os.path import basename -from pathlib import Path from typing import Tuple, Type, Optional from celery import signature, Task @@ -25,13 +19,11 @@ from packit_service.constants import ( COPR_API_SUCC_STATE, COPR_SRPM_CHROOT, - DOCS_URL, ) from packit_service.models import ( CoprBuildTargetModel, BuildStatus, ProjectEventModelType, - SRPMBuildModel, ) from packit_service.service.urls import get_copr_build_info_url, get_srpm_build_info_url from packit_service.utils import ( @@ -39,7 +31,6 @@ dump_package_config, elapsed_seconds, pr_labels_match_configuration, - download_file, ) from packit_service.worker.checker.abstract import Checker from packit_service.worker.checker.copr import ( @@ -79,7 +70,7 @@ GetCoprBuildJobHelperMixin, ConfigFromEventMixin, ) -from packit_service.worker.helpers.build import CoprBuildJobHelper +from packit_service.worker.helpers.open_scan_hub import OpenScanHubHelper from packit_service.worker.mixin import PackitAPIWithDownstreamMixin from packit_service.worker.reporting import BaseCommitStatus, DuplicateCheckMode from packit_service.worker.result import TaskResults @@ -368,13 +359,13 @@ def run(self): self.handle_testing_farm() if ( - not ScanHelper.osh_disabled() + not OpenScanHubHelper.osh_disabled() and self.db_project_event.type == ProjectEventModelType.pull_request and self.build.target == "fedora-rawhide-x86_64" and self.job_config.osh_diff_scan_after_copr_build ): try: - ScanHelper( + OpenScanHubHelper( copr_build_helper=self.copr_build_helper, build=self.build, ).handle_scan() @@ -509,196 +500,3 @@ def handle_testing_farm(self): "build_id": self.build.id, }, ).apply_async() - - -class ScanHelper: - def __init__( - self, copr_build_helper: CoprBuildJobHelper, build: CoprBuildTargetModel - ): - self.build = build - self.copr_build_helper = copr_build_helper - - @staticmethod - def osh_disabled() -> bool: - disabled = getenv("DISABLE_OPENSCANHUB", "False").lower() in ( - "true", - "t", - "yes", - "y", - "1", - ) - if disabled: - logger.info("OpenScanHub disabled via env var.") - return disabled - - def handle_scan(self): - """ - Try to find a job that can provide the base SRPM, - download both SRPM and base SRPM and trigger the scan in OpenScanHub. - """ - if not (base_build_job := self.find_base_build_job()): - logger.debug("No base build job needed for diff scan found in the config.") - return - - logger.info("Preparing to trigger scan in OpenScanHub...") - - if not (base_srpm_model := self.get_base_srpm_model(base_build_job)): - return - - srpm_model = self.build.get_srpm_build() - - with tempfile.TemporaryDirectory() as directory: - if not ( - paths := self.download_srpms(directory, base_srpm_model, srpm_model) - ): - return - - build_dashboard_url = get_copr_build_info_url(self.build.id) - - output = self.copr_build_helper.api.run_osh_build( - srpm_path=paths[1], - base_srpm=paths[0], - comment=f"Submitted via Packit Service for {build_dashboard_url}", - ) - - if not output: - logger.debug("Something went wrong, skipping the reporting.") - return - - logger.info("Scan submitted successfully.") - - response_dict = self.parse_dict_from_output(output) - - logger.debug(f"Parsed dict from output: {response_dict} ") - - if not (url := response_dict.get("url")): - logger.debug("It was not possible to get the URL from the response.") - return - - self.copr_build_helper._report( - state=BaseCommitStatus.success, - description=( - "Scan in OpenScanHub submitted successfully. Check the URL for more details." - ), - url=url, - check_names=["osh-diff-scan:fedora-rawhide-x86_64"], - markdown_content=( - "This is an experimental feature. Once the scan finishes, you can see the " - "new findings in the `added.html` in `Logs`. \n\n" - ":warning: You can see the list of known issues and also provide your feedback" - " [here](https://github.com/packit/packit/discussions/2371). \n\n" - "You can disable the scanning in your configuration by " - "setting `osh_diff_scan_after_copr_build` to `false`. For more information, " - f"see [docs]({DOCS_URL}/configuration#osh_diff_scan_after_copr_build)." - ), - ) - - @staticmethod - def parse_dict_from_output(output: str) -> dict: - json_pattern = r"\{.*?\}" - matches = re.findall(json_pattern, output, re.DOTALL) - - if not matches: - return {} - - json_str = matches[-1] - return json.loads(json_str) - - def find_base_build_job(self) -> Optional[JobConfig]: - """ - Find the job in the config that can provide the base build for the scan - (with `commit` trigger and same branch configured as the target PR branch). - """ - base_build_job = None - - for job in self.copr_build_helper.package_config.get_job_views(): - if ( - job.type in (JobType.copr_build, JobType.build) - and job.trigger == JobConfigTriggerType.commit - and ( - ( - job.branch - and job.branch - == self.copr_build_helper.pull_request_object.target_branch - ) - or ( - not job.branch - and self.copr_build_helper.project.default_branch - == self.copr_build_helper.pull_request_object.target_branch - ) - ) - ): - base_build_job = job - break - - return base_build_job - - def get_base_srpm_model( - self, base_build_job: JobConfig - ) -> Optional[SRPMBuildModel]: - """ - Get the SRPM build model of the latest successful Copr build - for the given job config. - """ - base_build_project_name = ( - self.copr_build_helper.job_project_for_commit_job_config(base_build_job) - ) - base_build_owner = self.copr_build_helper.job_owner_for_job_config( - base_build_job - ) - - def get_srpm_build(commit_sha): - logger.debug( - f"Searching for base build for {target_branch_commit} commit " - f"in {base_build_owner}/{base_build_project_name} Copr project in our DB. " - ) - - builds = CoprBuildTargetModel.get_all_by( - commit_sha=commit_sha, - project_name=base_build_project_name, - owner=base_build_owner, - target="fedora-rawhide-x86_64", - status=BuildStatus.success, - ) - try: - return next(iter(builds)).get_srpm_build() - except StopIteration: - return None - - target_branch_commit = ( - self.copr_build_helper.pull_request_object.target_branch_head_commit - ) - - if srpm_build := get_srpm_build(target_branch_commit): - return srpm_build - - for target_branch_commit in self.copr_build_helper.project.get_commits( - self.copr_build_helper.pull_request_object.target_branch - )[1:]: - if srpm_build := get_srpm_build(target_branch_commit): - return srpm_build - else: - logger.debug("No matching base build found in our DB.") - return None - - @staticmethod - def download_srpms( - directory: str, - base_srpm_model: SRPMBuildModel, - srpm_model: SRPMBuildModel, - ) -> Optional[tuple[Path, Path]]: - - def download_srpm(srpm_model: SRPMBuildModel) -> Optional[Path]: - srpm_path = Path(directory).joinpath(basename(srpm_model.url)) - if not download_file(srpm_model.url, srpm_path): - logger.info(f"Downloading of SRPM {srpm_model.url} was not successful.") - return None - return srpm_path - - if (base_srpm_path := download_srpm(base_srpm_model)) is None: - return None - - if (srpm_path := download_srpm(srpm_model)) is None: - return None - - return base_srpm_path, srpm_path diff --git a/packit_service/worker/handlers/open_scan_hub.py b/packit_service/worker/handlers/open_scan_hub.py new file mode 100644 index 000000000..791b5b5f5 --- /dev/null +++ b/packit_service/worker/handlers/open_scan_hub.py @@ -0,0 +1,170 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import logging +from typing import Tuple, Type, Union + +from packit.config import ( + JobType, +) +from packit_service.worker.checker.abstract import Checker +from packit_service.worker.events import ( + OpenScanHubTaskStartedEvent, + OpenScanHubTaskFinishedEvent, +) +from packit_service.worker.handlers.abstract import ( + RetriableJobHandler, + TaskName, + configured_as, + reacts_to, +) +from packit_service.worker.helpers.build import CoprBuildJobHelper +from packit_service.worker.helpers.open_scan_hub import OpenScanHubHelper + +from packit_service.worker.handlers.mixin import ( + ConfigFromEventMixin, +) +from packit_service.worker.result import TaskResults +from packit_service.worker.mixin import ( + LocalProjectMixin, + PackitAPIWithUpstreamMixin, +) +from packit_service.worker.checker.open_scan_hub import RawhideX86Target + +from packit_service.worker.reporting import BaseCommitStatus +from packit_service.models import OSHScanStatus + +logger = logging.getLogger(__name__) + + +class OpenScanHubAbstractHandler( + RetriableJobHandler, + LocalProjectMixin, + ConfigFromEventMixin, + PackitAPIWithUpstreamMixin, +): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.event: Union[ + OpenScanHubTaskFinishedEvent | OpenScanHubTaskStartedEvent + ] = self.data.to_event() + + @staticmethod + def get_checkers() -> Tuple[Type[Checker], ...]: + return (RawhideX86Target,) + + def get_helper(self) -> OpenScanHubHelper: + build_helper = CoprBuildJobHelper( + service_config=self.service_config, + package_config=self.package_config, + project=self.project, + metadata=self.data, + db_project_event=self.data.db_project_event, + job_config=self.job_config, + celery_task=self.celery_task, + ) + + scan_helper = OpenScanHubHelper( + copr_build_helper=build_helper, build=self.event.build + ) + return scan_helper + + def check_scan_and_build(self): + task_id = self.data.event_dict["task_id"] + if not self.event.scan or not self.event.build: + return TaskResults( + success=True, + details={ + "msg": f"Scan {task_id} not found or not associated with a Copr build" + }, + ) + elif not self.job_config: + return TaskResults( + success=True, + details={ + "msg": ( + "No job configuration found for OpenScanHub task" + f" in {self.project.repo}" + ) + }, + ) + + +@configured_as(job_type=JobType.copr_build) +@reacts_to(OpenScanHubTaskFinishedEvent) +class OpenScanHubTaskFinishedHandler( + OpenScanHubAbstractHandler, +): + event: OpenScanHubTaskFinishedEvent + task_name = TaskName.openscanhub_task_finished + + def run(self) -> TaskResults: + self.check_scan_and_build() + + if self.event.status == OpenScanHubTaskFinishedEvent.Status.success: + state = BaseCommitStatus.success + description = ( + "Scan in OpenScanHub is finished. Check the URL for more details." + ) + external_links = { + "Added issues": self.event.issues_added_url, + "Fixed issues": self.event.issues_fixed_url, + "Scan results": self.event.scan_results_url, + } + self.event.scan.set_status(OSHScanStatus.succeeded) + self.event.scan.set_issues_added_url(self.event.issues_added_url) + self.event.scan.set_issues_fixed_url(self.event.issues_fixed_url) + self.event.scan.set_scan_results_url(self.event.scan_results_url) + else: + state = BaseCommitStatus.neutral + description = ( + f"Scan in OpenScanHub is finished in a {self.event.status} state." + ) + external_links = {} + if self.event.status == OpenScanHubTaskFinishedEvent.Status.cancel: + self.event.scan.set_status(OSHScanStatus.canceled) + else: + self.event.scan.set_status(OSHScanStatus.failed) + + self.get_helper().report( + state=state, + description=description, + url=self.event.scan.url, + links_to_external_services=external_links, + ) + + return TaskResults( + success=True, + details={}, + ) + + +@configured_as(job_type=JobType.copr_build) +@reacts_to(OpenScanHubTaskStartedEvent) +class OpenScanHubTaskStartedHandler( + OpenScanHubAbstractHandler, +): + task_name = TaskName.openscanhub_task_started + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.event: OpenScanHubTaskStartedEvent = self.data.to_event() + + def run(self) -> TaskResults: + self.check_scan_and_build() + + state = BaseCommitStatus.running + description = "Scan in OpenScanHub has started." + self.event.scan.set_status(OSHScanStatus.running) + + self.get_helper().report( + state=state, + description=description, + url=self.event.scan.url, + ) + + return TaskResults( + success=True, + details={}, + ) diff --git a/packit_service/worker/helpers/open_scan_hub.py b/packit_service/worker/helpers/open_scan_hub.py new file mode 100644 index 000000000..567e4f0f3 --- /dev/null +++ b/packit_service/worker/helpers/open_scan_hub.py @@ -0,0 +1,243 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import json +import logging +import re +import tempfile +from os import getenv +from os.path import basename +from pathlib import Path +from typing import Optional, Dict + + +from packit.config import ( + JobConfig, + JobType, +) +from packit.config import JobConfigTriggerType +from packit_service.constants import ( + OPEN_SCAN_HUB_FEATURE_DESCRIPTION, +) +from packit_service.models import ( + CoprBuildTargetModel, + BuildStatus, + SRPMBuildModel, +) +from packit_service.service.urls import get_copr_build_info_url +from packit_service.utils import ( + download_file, +) +from packit_service.worker.helpers.build import CoprBuildJobHelper +from packit_service.worker.reporting import BaseCommitStatus + +logger = logging.getLogger(__name__) + + +class OpenScanHubHelper: + def __init__( + self, copr_build_helper: CoprBuildJobHelper, build: CoprBuildTargetModel + ): + self.build = build + self.copr_build_helper = copr_build_helper + + @staticmethod + def osh_disabled() -> bool: + disabled = getenv("DISABLE_OPENSCANHUB", "False").lower() in ( + "true", + "t", + "yes", + "y", + "1", + ) + if disabled: + logger.info("OpenScanHub disabled via env var.") + return disabled + + def handle_scan(self): + """ + Try to find a job that can provide the base SRPM, + download both SRPM and base SRPM and trigger the scan in OpenScanHub. + """ + if not (base_build_job := self.find_base_build_job()): + logger.debug("No base build job needed for diff scan found in the config.") + return + + logger.info("Preparing to trigger scan in OpenScanHub...") + + if not (base_srpm_model := self.get_base_srpm_model(base_build_job)): + return + + srpm_model = self.build.get_srpm_build() + + with tempfile.TemporaryDirectory() as directory: + if not ( + paths := self.download_srpms(directory, base_srpm_model, srpm_model) + ): + return + + build_dashboard_url = get_copr_build_info_url(self.build.id) + + output = self.copr_build_helper.api.run_osh_build( + srpm_path=paths[1], + base_srpm=paths[0], + comment=f"Submitted via Packit Service for {build_dashboard_url}", + ) + + if not output: + logger.debug("Something went wrong, skipping the reporting.") + return + + logger.info("Scan submitted successfully.") + + response_dict = self.parse_dict_from_output(output) + + logger.debug(f"Parsed dict from output: {response_dict} ") + + scan = None + if id := response_dict.get("id"): + scan = self.build.add_scan(task_id=id) + else: + logger.debug( + "It was not possible to get the Open Scan Hub task_id from the response." + ) + + if not (url := response_dict.get("url")): + logger.debug("It was not possible to get the URL from the response.") + return + if url and scan: + scan.set_url(url) + + self.report( + state=BaseCommitStatus.running, + description=( + "Scan in OpenScanHub submitted successfully. Check the URL for more details." + ), + url=url, + ) + + def report( + self, + state: BaseCommitStatus, + description: str, + url: str, + links_to_external_services: Optional[Dict[str, str]] = None, + ): + self.copr_build_helper._report( + state=state, + description=description, + url=url, + check_names=["osh-diff-scan:fedora-rawhide-x86_64"], + markdown_content=OPEN_SCAN_HUB_FEATURE_DESCRIPTION, + links_to_external_services=links_to_external_services, + ) + + @staticmethod + def parse_dict_from_output(output: str) -> dict: + json_pattern = r"\{.*?\}" + matches = re.findall(json_pattern, output, re.DOTALL) + + if not matches: + return {} + + json_str = matches[-1] + return json.loads(json_str) + + def find_base_build_job(self) -> Optional[JobConfig]: + """ + Find the job in the config that can provide the base build for the scan + (with `commit` trigger and same branch configured as the target PR branch). + """ + base_build_job = None + + for job in self.copr_build_helper.package_config.get_job_views(): + if ( + job.type in (JobType.copr_build, JobType.build) + and job.trigger == JobConfigTriggerType.commit + and ( + ( + job.branch + and job.branch + == self.copr_build_helper.pull_request_object.target_branch + ) + or ( + not job.branch + and self.copr_build_helper.project.default_branch + == self.copr_build_helper.pull_request_object.target_branch + ) + ) + ): + base_build_job = job + break + + return base_build_job + + def get_base_srpm_model( + self, base_build_job: JobConfig + ) -> Optional[SRPMBuildModel]: + """ + Get the SRPM build model of the latest successful Copr build + for the given job config. + """ + base_build_project_name = ( + self.copr_build_helper.job_project_for_commit_job_config(base_build_job) + ) + base_build_owner = self.copr_build_helper.job_owner_for_job_config( + base_build_job + ) + + def get_srpm_build(commit_sha): + logger.debug( + f"Searching for base build for {target_branch_commit} commit " + f"in {base_build_owner}/{base_build_project_name} Copr project in our DB. " + ) + + builds = CoprBuildTargetModel.get_all_by( + commit_sha=commit_sha, + project_name=base_build_project_name, + owner=base_build_owner, + target="fedora-rawhide-x86_64", + status=BuildStatus.success, + ) + try: + return next(iter(builds)).get_srpm_build() + except StopIteration: + return None + + target_branch_commit = ( + self.copr_build_helper.pull_request_object.target_branch_head_commit + ) + + if srpm_build := get_srpm_build(target_branch_commit): + return srpm_build + + for target_branch_commit in self.copr_build_helper.project.get_commits( + self.copr_build_helper.pull_request_object.target_branch + )[1:]: + if srpm_build := get_srpm_build(target_branch_commit): + return srpm_build + else: + logger.debug("No matching base build found in our DB.") + return None + + @staticmethod + def download_srpms( + directory: str, + base_srpm_model: SRPMBuildModel, + srpm_model: SRPMBuildModel, + ) -> Optional[tuple[Path, Path]]: + + def download_srpm(srpm_model: SRPMBuildModel) -> Optional[Path]: + srpm_path = Path(directory).joinpath(basename(srpm_model.url)) + if not download_file(srpm_model.url, srpm_path): + logger.info(f"Downloading of SRPM {srpm_model.url} was not successful.") + return None + return srpm_path + + if (base_srpm_path := download_srpm(base_srpm_model)) is None: + return None + + if (srpm_path := download_srpm(srpm_model)) is None: + return None + + return base_srpm_path, srpm_path diff --git a/packit_service/worker/parser.py b/packit_service/worker/parser.py index 515b338cf..a6ff0fa6b 100644 --- a/packit_service/worker/parser.py +++ b/packit_service/worker/parser.py @@ -53,6 +53,8 @@ ReleaseGitlabEvent, TagPushGitlabEvent, VMImageBuildResultEvent, + OpenScanHubTaskFinishedEvent, + OpenScanHubTaskStartedEvent, ) from packit_service.worker.events.enums import ( GitlabEventAction, @@ -115,6 +117,8 @@ def parse_event( TagPushGitlabEvent, VMImageBuildResultEvent, AnityaVersionUpdateEvent, + OpenScanHubTaskFinishedEvent, + OpenScanHubTaskStartedEvent, ] ]: """ @@ -160,6 +164,8 @@ def parse_event( Parser.parse_gitlab_release_event, Parser.parse_gitlab_tag_push_event, Parser.parse_anitya_version_update_event, + Parser.parse_openscanhub_task_finished_event, + Parser.parse_openscanhub_task_started_event, ), ): if response: @@ -1651,6 +1657,56 @@ def parse_anitya_version_update_event(event) -> Optional[AnityaVersionUpdateEven anitya_project_name=anitya_project_name, ) + @staticmethod + def parse_openscanhub_task_finished_event( + event, + ) -> Optional[OpenScanHubTaskFinishedEvent]: + + if "openscanhub.task.finished" not in event.get("topic", ""): + return None + + task_id = event.get("task_id") + status = event.get("status") + logger.info(f"OpenScanHub task: {task_id} finished with status {status}.") + + event = OpenScanHubTaskFinishedEvent( + task_id=task_id, + status=status, + issues_added_url=event.get("added.js", ""), + issues_fixed_url=event.get("fixed.js", ""), + scan_results_url=event.get("scan-results.js", ""), + ) + + if not event.build: + logger.warning( + "OpenScanHub task.finished is missing association with build. " + "Package config can not be resolved without it. " + "Skipping the event." + ) + return None + return event + + @staticmethod + def parse_openscanhub_task_started_event( + event, + ) -> Optional[OpenScanHubTaskStartedEvent]: + + if "openscanhub.task.started" not in event.get("topic", ""): + return None + + task_id = event.get("task_id") + logger.info(f"OpenScanHub task: {task_id} started.") + + event = OpenScanHubTaskStartedEvent(task_id=task_id) + if not event.build: + logger.warning( + "OpenScanHub task.started is missing association with build. " + "Package config can not be resolved without it. " + "Skipping the event." + ) + return None + return event + # The .__func__ are needed for Python < 3.10 MAPPING = { "github": { @@ -1684,6 +1740,12 @@ def parse_anitya_version_update_event(event) -> Optional[AnityaVersionUpdateEven "org.release-monitoring.prod.anitya.project.version.update.v2": ( parse_anitya_version_update_event.__func__ # type: ignore ), + "openscanhub.task.started": ( + parse_openscanhub_task_started_event.__func__, # type: ignore + ), + "openscanhub.task.finished": ( + parse_openscanhub_task_finished_event.__func__, # type: ignore + ), }, "testing-farm": { "results": parse_testing_farm_results_event.__func__, # type: ignore diff --git a/packit_service/worker/tasks.py b/packit_service/worker/tasks.py index 26a82a50d..81f1e7ea1 100644 --- a/packit_service/worker/tasks.py +++ b/packit_service/worker/tasks.py @@ -59,6 +59,8 @@ TestingFarmResultsHandler, VMImageBuildHandler, VMImageBuildResultHandler, + OpenScanHubTaskFinishedHandler, + OpenScanHubTaskStartedHandler, ) from packit_service.worker.handlers.abstract import TaskName from packit_service.worker.handlers.bodhi import ( @@ -608,6 +610,32 @@ def run_tag_into_sidetag_handler( return get_handlers_task_results(handler.run_job(), event) +@celery_app.task(bind=True, name=TaskName.openscanhub_task_finished, base=TaskWithRetry) +def run_openscanhub_task_finished_handler( + self, event: dict, package_config: dict, job_config: dict +): + handler = OpenScanHubTaskFinishedHandler( + package_config=load_package_config(package_config), + job_config=load_job_config(job_config), + event=event, + celery_task=self, + ) + return get_handlers_task_results(handler.run_job(), event) + + +@celery_app.task(bind=True, name=TaskName.openscanhub_task_started, base=TaskWithRetry) +def run_openscanhub_task_started_handler( + self, event: dict, package_config: dict, job_config: dict +): + handler = OpenScanHubTaskStartedHandler( + package_config=load_package_config(package_config), + job_config=load_job_config(job_config), + event=event, + celery_task=self, + ) + return get_handlers_task_results(handler.run_job(), event) + + def get_handlers_task_results(results: dict, event: dict) -> dict: # include original event to provide more info return {"job": results, "event": event} diff --git a/tests/conftest.py b/tests/conftest.py index 3997f3f70..cbd78ba07 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -339,7 +339,7 @@ def add_pull_request_event_with_sha_123456(): id=123, ) db_project_event = ( - flexmock(type=ProjectEventModelType.pull_request) + flexmock(type=ProjectEventModelType.pull_request, commit_sha="123456") .should_receive("get_project_event_object") .and_return(db_project_object) .mock() diff --git a/tests/data/fedmsg/open_scan_hub_task_finished.json b/tests/data/fedmsg/open_scan_hub_task_finished.json new file mode 100644 index 000000000..fc3150866 --- /dev/null +++ b/tests/data/fedmsg/open_scan_hub_task_finished.json @@ -0,0 +1,9 @@ +{ + "added.js": "http://openscanhub.fedoraproject.org/task/15649/log/added.js?format=raw", + "fixed.js": "http://openscanhub.fedoraproject.org/task/15649/log/fixed.js?format=raw", + "scan-results.js": "http://openscanhub.fedoraproject.org/task/15649/log/gvisor-tap-vsock-0.7.5-1.20241007054606793155.pr405.23.g829aafd6/scan-results.js?format=raw", + "task_id": 15649, + "status": "success", + "topic": "org.fedoraproject.prod.openscanhub.task.finished", + "timestamp": 1728297198.032705 +} diff --git a/tests/data/fedmsg/open_scan_hub_task_started.json b/tests/data/fedmsg/open_scan_hub_task_started.json new file mode 100644 index 000000000..cc5eaff7e --- /dev/null +++ b/tests/data/fedmsg/open_scan_hub_task_started.json @@ -0,0 +1,5 @@ +{ + "task_id": 15649, + "topic": "org.fedoraproject.prod.openscanhub.task.started", + "timestamp": 1728297198.032705 +} diff --git a/tests/unit/events/test_open_scan_hub.py b/tests/unit/events/test_open_scan_hub.py new file mode 100644 index 000000000..135ef3b95 --- /dev/null +++ b/tests/unit/events/test_open_scan_hub.py @@ -0,0 +1,121 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import json +import pytest +import datetime +from flexmock import flexmock + +from packit.config import ( + JobConfig, + CommonPackageConfig, + PackageConfig, + JobType, + JobConfigTriggerType, +) +from packit_service.worker.events import ( + OpenScanHubTaskFinishedEvent, + OpenScanHubTaskStartedEvent, +) +from packit_service.worker.parser import Parser +from packit_service.models import OSHScanModel + +from tests.spellbook import DATA_DIR + + +@pytest.fixture() +def openscanhub_task_finished_event(): + with open(DATA_DIR / "fedmsg" / "open_scan_hub_task_finished.json") as outfile: + return json.load(outfile) + + +@pytest.fixture() +def openscanhub_task_started_event(): + with open(DATA_DIR / "fedmsg" / "open_scan_hub_task_started.json") as outfile: + return json.load(outfile) + + +@pytest.fixture() +def scan_config_and_db(add_pull_request_event_with_sha_123456): + db_project_object, db_project_event = add_pull_request_event_with_sha_123456 + db_build = ( + flexmock( + build_id="55", + status="success", + build_submitted_time=datetime.datetime.utcnow(), + target="the-target", + owner="the-owner", + project_name="the-namespace-repo_name-5", + commit_sha="123456", + project_event=flexmock(), + srpm_build=flexmock(url=None) + .should_receive("set_url") + .with_args("https://some.host/my.srpm") + .mock(), + ) + .should_receive("get_project_event_object") + .and_return(db_project_object) + .mock() + .should_receive("get_project_event_model") + .and_return(db_project_event) + .mock() + ) + flexmock(OpenScanHubTaskFinishedEvent).should_receive( + "get_packages_config" + ).and_return( + PackageConfig( + jobs=[ + JobConfig( + type=JobType.copr_build, + trigger=JobConfigTriggerType.commit, + packages={ + "package": CommonPackageConfig( + _targets=["fedora-rawhide-x86_64"], + ) + }, + ) + ], + packages={"package": CommonPackageConfig()}, + ) + ) + flexmock(OSHScanModel).should_receive("get_by_task_id").and_return( + flexmock(copr_build_target=db_build) + ) + + +def test_parse_openscanhub_task_finished( + openscanhub_task_finished_event, scan_config_and_db +): + event_object = Parser.parse_event(openscanhub_task_finished_event) + + assert isinstance(event_object, OpenScanHubTaskFinishedEvent) + assert event_object.task_id == 15649 + assert ( + event_object.issues_added_url + == "http://openscanhub.fedoraproject.org/task/15649/log/added.js?format=raw" + ) + assert ( + event_object.issues_fixed_url + == "http://openscanhub.fedoraproject.org/task/15649/log/fixed.js?format=raw" + ) + assert event_object.scan_results_url == ( + "http://openscanhub.fedoraproject.org/task/15649/log/gvisor-tap-vsock" + "-0.7.5-1.20241007054606793155.pr405.23.g829aafd6/scan-results.js?format=raw" + ) + assert event_object.db_project_event + assert event_object.db_project_object + assert event_object.project + assert json.dumps(event_object.get_dict()) + + +def test_parse_openscanhub_task_started( + openscanhub_task_started_event, scan_config_and_db +): + event_object = Parser.parse_event(openscanhub_task_started_event) + + assert isinstance(event_object, OpenScanHubTaskStartedEvent) + assert event_object.task_id == 15649 + assert event_object.db_project_event + assert event_object.db_project_object + assert event_object.project + assert json.dumps(event_object.get_dict()) diff --git a/tests/unit/test_open_scan_hub.py b/tests/unit/test_open_scan_hub.py new file mode 100644 index 000000000..ea597a483 --- /dev/null +++ b/tests/unit/test_open_scan_hub.py @@ -0,0 +1,386 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import datetime +import pytest +import json +from flexmock import flexmock +from celery.canvas import group as celery_group + +from packit.api import PackitAPI +from packit.config import ( + JobType, + JobConfigTriggerType, + PackageConfig, + JobConfig, + CommonPackageConfig, +) +from packit_service.models import ( + CoprBuildTargetModel, + ProjectEventModelType, + BuildStatus, + OSHScanModel, +) +from packit_service.worker.tasks import ( + run_openscanhub_task_finished_handler, + run_openscanhub_task_started_handler, +) +from packit_service.worker.jobs import SteveJobs +from packit_service.worker.monitoring import Pushgateway +from packit_service.worker.reporting import BaseCommitStatus + +from packit_service.worker.events import ( + AbstractCoprBuildEvent, + OpenScanHubTaskFinishedEvent, + OpenScanHubTaskStartedEvent, +) +from packit_service.worker.helpers import open_scan_hub +from packit_service.worker.handlers.copr import OpenScanHubHelper +from packit_service.worker.helpers.build import CoprBuildJobHelper + +from tests.spellbook import DATA_DIR, get_parameters_from_results + + +@pytest.fixture() +def openscanhub_task_finished_event(): + with open(DATA_DIR / "fedmsg" / "open_scan_hub_task_finished.json") as outfile: + return json.load(outfile) + + +@pytest.fixture() +def openscanhub_task_started_event(): + with open(DATA_DIR / "fedmsg" / "open_scan_hub_task_started.json") as outfile: + return json.load(outfile) + + +@pytest.fixture() +def prepare_openscanhub_db_and_handler( + add_pull_request_event_with_sha_123456, +): + db_project_object, db_project_event = add_pull_request_event_with_sha_123456 + db_build = ( + flexmock( + build_id="55", + status="success", + build_submitted_time=datetime.datetime.utcnow(), + target="the-target", + owner="the-owner", + project_name="the-namespace-repo_name-5", + commit_sha="123456", + project_event=flexmock(), + srpm_build=flexmock(url=None) + .should_receive("set_url") + .with_args("https://some.host/my.srpm") + .mock(), + ) + .should_receive("get_project_event_object") + .and_return(db_project_object) + .mock() + .should_receive("get_project_event_model") + .and_return(db_project_event) + .mock() + ) + + flexmock(celery_group).should_receive("apply_async") + scan_mock = flexmock( + copr_build_target=db_build, + url="https://openscanhub.fedoraproject.org/task/17514/", + set_issues_added_url=lambda _: None, + set_issues_fixed_url=lambda _: None, + set_scan_results_url=lambda _: None, + ) + flexmock(OSHScanModel).should_receive("get_by_task_id").and_return(scan_mock) + flexmock(Pushgateway).should_receive("push").and_return() + yield scan_mock + + +@pytest.mark.parametrize( + "build_models", + [ + [("abcdef", [flexmock(get_srpm_build=lambda: flexmock(url="base-srpm-url"))])], + [ + ("abcdef", []), + ( + "fedcba", + [flexmock(get_srpm_build=lambda: flexmock(url="base-srpm-url"))], + ), + ], + ], +) +def test_handle_scan(build_models): + srpm_mock = flexmock(url="https://some-url/my-srpm.src.rpm") + flexmock(AbstractCoprBuildEvent).should_receive("from_event_dict").and_return( + flexmock(chroot="fedora-rawhide-x86_64", build_id="123", pr_id=12) + ) + flexmock(open_scan_hub).should_receive("download_file").twice().and_return(True) + + for commit_sha, models in build_models: + flexmock(CoprBuildTargetModel).should_receive("get_all_by").with_args( + commit_sha=commit_sha, + project_name="commit-project", + owner="user-123", + target="fedora-rawhide-x86_64", + status=BuildStatus.success, + ).and_return(models).once() + + flexmock(PackitAPI).should_receive("run_osh_build").once().and_return( + 'some\nmultiline\noutput\n{"id": 123}\nand\nmore\n{"url": "scan-url"}\n' + ) + + flexmock(CoprBuildJobHelper).should_receive("_report") + package_config = flexmock( + get_job_views=lambda: [ + flexmock( + type=JobType.copr_build, + trigger=JobConfigTriggerType.commit, + branch="main", + project="commit-project", + owner="user-123", + ) + ] + ) + + project = flexmock( + get_pr=lambda pr_id: flexmock( + target_branch="main", target_branch_head_commit="abcdef" + ), + get_commits=lambda ref: ["abcdef", "fedcba"], + ) + + OpenScanHubHelper( + build=flexmock( + id=1, + get_srpm_build=lambda: srpm_mock, + target="fedora-rawhide-x86_64", + get_project_event_model=lambda: flexmock( + type=ProjectEventModelType.pull_request, + get_project_event_object=lambda: flexmock(), + ), + ), + copr_build_helper=CoprBuildJobHelper( + service_config=flexmock(), + package_config=package_config, + project=project, + metadata=flexmock(pr_id=12), + db_project_event=flexmock(get_project_event_object=lambda: None), + job_config=flexmock(), + ), + ).handle_scan() + + +@pytest.mark.parametrize( + "job_config_type,job_config_trigger,job_config_targets,scan_status,num_of_handlers", + [ + ( + JobType.copr_build, + JobConfigTriggerType.commit, + ["fedora-rawhide-x86_64"], + OpenScanHubTaskFinishedEvent.Status.success, + 0, + ), + ( + JobType.copr_build, + JobConfigTriggerType.pull_request, + ["fedora-stable"], + OpenScanHubTaskFinishedEvent.Status.success, + 0, + ), + ( + JobType.copr_build, + JobConfigTriggerType.pull_request, + ["fedora-rawhide-x86_64"], + OpenScanHubTaskFinishedEvent.Status.success, + 1, + ), + ( + JobType.copr_build, + JobConfigTriggerType.pull_request, + ["fedora-rawhide-x86_64"], + OpenScanHubTaskFinishedEvent.Status.fail, + 1, + ), + ( + JobType.copr_build, + JobConfigTriggerType.pull_request, + ["fedora-rawhide-x86_64"], + OpenScanHubTaskFinishedEvent.Status.cancel, + 1, + ), + ( + JobType.copr_build, + JobConfigTriggerType.commit, + ["fedora-rawhide-x86_64"], + OpenScanHubTaskFinishedEvent.Status.interrupt, + 0, + ), + ], +) +def test_handle_scan_task_finished( + openscanhub_task_finished_event, + prepare_openscanhub_db_and_handler, + job_config_type, + job_config_trigger, + job_config_targets, + scan_status, + num_of_handlers, +): + + flexmock(OpenScanHubTaskFinishedEvent).should_receive( + "get_packages_config" + ).and_return( + PackageConfig( + jobs=[ + JobConfig( + type=job_config_type, + trigger=job_config_trigger, + packages={ + "package": CommonPackageConfig( + _targets=job_config_targets, + specfile_path="test.spec", + ) + }, + ), + ], + packages={"package": CommonPackageConfig()}, + ) + ) + + scan_mock = prepare_openscanhub_db_and_handler + openscanhub_task_finished_event["status"] = scan_status + processing_results = SteveJobs().process_message(openscanhub_task_finished_event) + assert len(processing_results) == num_of_handlers + + if processing_results: + + if scan_status == OpenScanHubTaskFinishedEvent.Status.success: + state = BaseCommitStatus.success + description = ( + "Scan in OpenScanHub is finished. Check the URL for more details." + ) + flexmock(scan_mock).should_receive("set_status").with_args( + "succeeded" + ).once() + links_to_external_services = { + "Added issues": ( + "http://openscanhub.fedoraproject.org/task/15649/log/added.js" + "?format=raw" + ), + "Fixed issues": ( + "http://openscanhub.fedoraproject.org/task/15649/log/fixed.js" + "?format=raw" + ), + "Scan results": ( + "http://openscanhub.fedoraproject.org/task/15649/log/gvisor-tap-vsock-" + "0.7.5-1.20241007054606793155.pr405.23.g829aafd6/scan-results.js?format=raw" + ), + } + elif scan_status == OpenScanHubTaskFinishedEvent.Status.cancel: + state = BaseCommitStatus.neutral + description = f"Scan in OpenScanHub is finished in a {scan_status} state." + links_to_external_services = {} + flexmock(scan_mock).should_receive("set_status").with_args( + "canceled" + ).once() + else: + state = BaseCommitStatus.neutral + description = f"Scan in OpenScanHub is finished in a {scan_status} state." + links_to_external_services = {} + flexmock(scan_mock).should_receive("set_status").with_args("failed").once() + if num_of_handlers == 1: + # one handler is always skipped because it is for fedora-stable -> + # no rawhide build + flexmock(OpenScanHubHelper).should_receive("report").with_args( + state=state, + description=description, + url="https://openscanhub.fedoraproject.org/task/17514/", + links_to_external_services=links_to_external_services, + ).once().and_return() + + for sub_results in processing_results: + event_dict, job, job_config, package_config = get_parameters_from_results( + [sub_results] + ) + assert json.dumps(event_dict) + + run_openscanhub_task_finished_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + + +@pytest.mark.parametrize( + "job_config_type,job_config_trigger,job_config_targets,num_of_handlers", + [ + ( + JobType.copr_build, + JobConfigTriggerType.commit, + ["fedora-rawhide-x86_64"], + 0, + ), + ( + JobType.copr_build, + JobConfigTriggerType.pull_request, + ["fedora-stable"], + 0, + ), + ( + JobType.copr_build, + JobConfigTriggerType.pull_request, + ["fedora-rawhide-x86_64"], + 1, + ), + ], +) +def test_handle_scan_task_started( + openscanhub_task_started_event, + prepare_openscanhub_db_and_handler, + job_config_type, + job_config_trigger, + job_config_targets, + num_of_handlers, +): + flexmock(OpenScanHubTaskStartedEvent).should_receive( + "get_packages_config" + ).and_return( + PackageConfig( + jobs=[ + JobConfig( + type=job_config_type, + trigger=job_config_trigger, + packages={ + "package": CommonPackageConfig( + _targets=job_config_targets, + specfile_path="test.spec", + ) + }, + ), + ], + packages={"package": CommonPackageConfig()}, + ) + ) + + scan_mock = prepare_openscanhub_db_and_handler + processing_results = SteveJobs().process_message(openscanhub_task_started_event) + assert len(processing_results) == num_of_handlers + + if processing_results: + if num_of_handlers == 1: + flexmock(scan_mock).should_receive("set_status").with_args("running").once() + flexmock(OpenScanHubHelper).should_receive("report").with_args( + state=BaseCommitStatus.running, + description="Scan in OpenScanHub has started.", + url="https://openscanhub.fedoraproject.org/task/17514/", + ).once().and_return() + + for sub_results in processing_results: + event_dict, job, job_config, package_config = get_parameters_from_results( + [sub_results] + ) + assert json.dumps(event_dict) + + run_openscanhub_task_started_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) diff --git a/tests/unit/test_scan.py b/tests/unit/test_scan.py deleted file mode 100644 index 4962ddc5a..000000000 --- a/tests/unit/test_scan.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright Contributors to the Packit project. -# SPDX-License-Identifier: MIT - -import pytest -from flexmock import flexmock - -from packit.api import PackitAPI -from packit.config import JobType, JobConfigTriggerType -from packit_service.models import ( - CoprBuildTargetModel, - ProjectEventModelType, - BuildStatus, -) -from packit_service.worker.events import AbstractCoprBuildEvent -from packit_service.worker.handlers import copr -from packit_service.worker.handlers.copr import ScanHelper -from packit_service.worker.helpers.build import CoprBuildJobHelper - - -@pytest.mark.parametrize( - "build_models", - [ - [("abcdef", [flexmock(get_srpm_build=lambda: flexmock(url="base-srpm-url"))])], - [ - ("abcdef", []), - ( - "fedcba", - [flexmock(get_srpm_build=lambda: flexmock(url="base-srpm-url"))], - ), - ], - ], -) -def test_handle_scan(build_models): - srpm_mock = flexmock(url="https://some-url/my-srpm.src.rpm") - flexmock(AbstractCoprBuildEvent).should_receive("from_event_dict").and_return( - flexmock(chroot="fedora-rawhide-x86_64", build_id="123", pr_id=12) - ) - flexmock(copr).should_receive("download_file").twice().and_return(True) - - for commit_sha, models in build_models: - flexmock(CoprBuildTargetModel).should_receive("get_all_by").with_args( - commit_sha=commit_sha, - project_name="commit-project", - owner="user-123", - target="fedora-rawhide-x86_64", - status=BuildStatus.success, - ).and_return(models).once() - - flexmock(PackitAPI).should_receive("run_osh_build").once().and_return( - 'some\nmultiline\noutput\n{"id": 123}\nand\nmore\n{"url": "scan-url"}\n' - ) - - flexmock(CoprBuildJobHelper).should_receive("_report") - package_config = flexmock( - get_job_views=lambda: [ - flexmock( - type=JobType.copr_build, - trigger=JobConfigTriggerType.commit, - branch="main", - project="commit-project", - owner="user-123", - ) - ] - ) - - project = flexmock( - get_pr=lambda pr_id: flexmock( - target_branch="main", target_branch_head_commit="abcdef" - ), - get_commits=lambda ref: ["abcdef", "fedcba"], - ) - - ScanHelper( - build=flexmock( - id=1, - get_srpm_build=lambda: srpm_mock, - target="fedora-rawhide-x86_64", - get_project_event_model=lambda: flexmock( - type=ProjectEventModelType.pull_request, - get_project_event_object=lambda: flexmock(), - ), - ), - copr_build_helper=CoprBuildJobHelper( - service_config=flexmock(), - package_config=package_config, - project=project, - metadata=flexmock(pr_id=12), - db_project_event=flexmock(get_project_event_object=lambda: None), - job_config=flexmock(), - ), - ).handle_scan() diff --git a/tests_openshift/conftest.py b/tests_openshift/conftest.py index 19fb42a7b..327847c42 100644 --- a/tests_openshift/conftest.py +++ b/tests_openshift/conftest.py @@ -49,6 +49,7 @@ BodhiUpdateTargetModel, BodhiUpdateGroupModel, SyncReleasePullRequestModel, + OSHScanModel, ) from packit_service.worker.events import InstallationEvent @@ -180,6 +181,7 @@ def global_service_config(): def clean_db(): with sa_session_transaction() as session: session.query(SourceGitPRDistGitPRModel).delete() + session.query(OSHScanModel).delete() session.query(AllowlistModel).delete() session.query(GithubInstallationModel).delete() @@ -2431,3 +2433,14 @@ def multiple_bodhi_update_runs(branch_project_event_model): bodhi_update_group=group, ), ] + + +@pytest.fixture() +def a_scan(a_copr_build_for_pr): + scan = a_copr_build_for_pr.add_scan(123) + scan.status = "success" + scan.url = "task url" + scan.issues_added_url = "added issues" + scan.issues_fixed_url = "fixed issues" + scan.scan_results_url = "results" + yield scan diff --git a/tests_openshift/database/test_models.py b/tests_openshift/database/test_models.py index e9674c0cc..1d95f133e 100644 --- a/tests_openshift/database/test_models.py +++ b/tests_openshift/database/test_models.py @@ -33,6 +33,7 @@ Session, BuildStatus, SyncReleaseJobType, + OSHScanModel, ) from tests_openshift.conftest import SampleValues @@ -1065,3 +1066,19 @@ def test_project_event_get_older_than_with_packages_config( ) == 0 ) + + +def test_create_scan(clean_before_and_after, a_scan): + assert a_scan.task_id == 123 + assert a_scan.status == "success" + assert a_scan.url == "task url" + assert a_scan.issues_added_url == "added issues" + assert a_scan.issues_fixed_url == "fixed issues" + assert a_scan.scan_results_url == "results" + assert a_scan.copr_build_target.build_id == "123456" + + +def test_add_scan_to_copr_build(clean_before_and_after, a_copr_build_for_pr): + a_copr_build_for_pr.add_scan(123) + scan = OSHScanModel.get_by_task_id(123) + assert scan.task_id == 123