Receive and save OpenScanHub task finish event (#2580)

Receive and save OpenScanHub task finish event

Fixes #2543
Merge after:

packit/packit-service-fedmsg#113
packit/deployment#603

Reviewed-by: Laura Barcziová
Reviewed-by: Maja Massarini
Reviewed-by: Matej Focko
Reviewed-by: Siteshwar Vashisht
Reviewed-by: Kamil Dudka <[email protected]>

alembic/versions/242a0cdb6d23_create_oshscanmodel.py

@@ -0,0 +1,57 @@
+"""Create OSHScanModel
+
+Revision ID: 242a0cdb6d23
+Revises: f376be1907e1
+Create Date: 2024-10-18 08:10:35.335510
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "242a0cdb6d23"
+down_revision = "f376be1907e1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "scans",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("task_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "status",
+            sa.Enum(
+                "pending",
+                "running",
+                "succeeded",
+                "canceled",
+                "failed",
+                name="oshscanstatus",
+            ),
+            nullable=True,
+        ),
+        sa.Column("url", sa.String(), nullable=True),
+        sa.Column("issues_added_url", sa.String(), nullable=True),
+        sa.Column("issues_fixed_url", sa.String(), nullable=True),
+        sa.Column("scan_results_url", sa.String(), nullable=True),
+        sa.Column("copr_build_target_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["copr_build_target_id"],
+            ["copr_build_targets.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("copr_build_target_id"),
+        sa.UniqueConstraint("task_id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("scans")
+    # ### end Alembic commands ###

packit_service/constants.py

@@ -302,3 +302,13 @@ def from_number(number: int):
 USAGE_PAST_YEAR_DATE_STR = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d")
 USAGE_DATE_IN_THE_PAST = USAGE_CURRENT_DATE.replace(year=USAGE_CURRENT_DATE.year - 100)
 USAGE_DATE_IN_THE_PAST_STR = USAGE_DATE_IN_THE_PAST.strftime("%Y-%m-%d")
+
+OPEN_SCAN_HUB_FEATURE_DESCRIPTION = (
+    "This is an experimental feature. Once the scan finishes, you can see the "
+    "new findings in the `added.html` in `Logs`. \n\n"
+    ":warning: You can see the list of known issues and also provide your feedback"
+    " [here](https://github.com/packit/packit/discussions/2371). \n\n"
+    "You can disable the scanning in your configuration by "
+    "setting `osh_diff_scan_after_copr_build` to `false`. For more information, "
+    f"see [docs]({DOCS_URL}/configuration#osh_diff_scan_after_copr_build)."
+)

packit_service/models.py

@@ -1970,6 +1970,8 @@ class CoprBuildTargetModel(GroupAndTargetModelConnector, Base):
         "CoprBuildGroupModel", back_populates="copr_build_targets"
     )
 
+    scan = relationship("OSHScanModel", back_populates="copr_build_target")
+
     def set_built_packages(self, built_packages):
         with sa_session_transaction(commit=True) as session:
             self.built_packages = built_packages
@@ -2195,6 +2197,13 @@ def __repr__(self):
             f"build_submitted_time={self.build_submitted_time})"
         )
 
+    def add_scan(self, task_id: int) -> "OSHScanModel":
+        with sa_session_transaction(commit=True) as session:
+            scan = OSHScanModel.get_or_create(task_id)
+            scan.copr_build_target = self
+            session.add(scan)
+            return scan
+
 
 class KojiBuildGroupModel(ProjectAndEventsConnector, GroupModel, Base):
     __tablename__ = "koji_build_groups"
@@ -3824,6 +3833,74 @@ def get_by_koji_name(cls, koji_name: str) -> Optional["SidetagModel"]:
             return session.query(SidetagModel).filter_by(koji_name=koji_name).first()
 
 
+class OSHScanStatus(str, enum.Enum):
+    """An enum of all possible build statuses"""
+
+    pending = "pending"
+    running = "running"
+    succeeded = "succeeded"
+    failed = "failed"
+    canceled = "canceled"
+
+
+class OSHScanModel(Base):
+    __tablename__ = "scans"
+    id = Column(Integer, primary_key=True)
+    task_id = Column(Integer, unique=True)  # open scan hub id
+    status = Column(Enum(OSHScanStatus))
+    url = Column(String)
+    issues_added_url = Column(String)
+    issues_fixed_url = Column(String)
+    scan_results_url = Column(String)
+    copr_build_target_id = Column(
+        Integer, ForeignKey("copr_build_targets.id"), unique=True
+    )
+    copr_build_target = relationship(
+        "CoprBuildTargetModel", back_populates="scan", uselist=False
+    )
+
+    @classmethod
+    def get_or_create(cls, task_id: int) -> "OSHScanModel":
+        with sa_session_transaction(commit=True) as session:
+            scan = cls.get_by_task_id(task_id)
+            if not scan:
+                scan = cls()
+                scan.task_id = task_id
+                scan.status = OSHScanStatus.pending
+                session.add(scan)
+            return scan
+
+    def set_status(self, status: OSHScanStatus) -> None:
+        with sa_session_transaction(commit=True) as session:
+            self.status = status
+            session.add(self)
+
+    def set_url(self, url: str) -> None:
+        with sa_session_transaction(commit=True) as session:
+            self.url = url
+            session.add(self)
+
+    def set_issues_added_url(self, issues_added_url: str) -> None:
+        with sa_session_transaction(commit=True) as session:
+            self.issues_added_url = issues_added_url
+            session.add(self)
+
+    def set_issues_fixed_url(self, issues_fixed_url: str) -> None:
+        with sa_session_transaction(commit=True) as session:
+            self.issues_fixed_url = issues_fixed_url
+            session.add(self)
+
+    def set_scan_results_url(self, scan_results_url: str) -> None:
+        with sa_session_transaction(commit=True) as session:
+            self.scan_results_url = scan_results_url
+            session.add(self)
+
+    @classmethod
+    def get_by_task_id(cls, task_id: int) -> Optional["OSHScanModel"]:
+        with sa_session_transaction() as session:
+            return session.query(cls).filter_by(task_id=task_id).first()
+
+
 @cached(cache=TTLCache(maxsize=2048, ttl=(60 * 60 * 24)))
 def get_usage_data(datetime_from=None, datetime_to=None, top=10) -> dict:
     """

packit_service/worker/allowlist.py

@@ -41,6 +41,8 @@
     ReleaseEvent,
     TestingFarmResultsEvent,
     CheckRerunEvent,
+    OpenScanHubTaskFinishedEvent,
+    OpenScanHubTaskStartedEvent,
 )
 from packit_service.worker.events.gitlab import ReleaseGitlabEvent
 from packit_service.worker.events.koji import KojiBuildEvent, KojiBuildTagEvent
@@ -485,6 +487,8 @@ def check_and_report(
                 KojiBuildTagEvent,
                 CheckRerunEvent,
                 NewHotnessUpdateEvent,
+                OpenScanHubTaskFinishedEvent,
+                OpenScanHubTaskStartedEvent,
             ): self._check_unchecked_event,
             (
                 ReleaseEvent,

packit_service/worker/checker/open_scan_hub.py

@@ -0,0 +1,26 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+import logging
+
+from packit.config import (
+    aliases,
+)
+from packit_service.worker.checker.abstract import Checker
+
+logger = logging.getLogger(__name__)
+
+
+class RawhideX86Target(
+    Checker,
+):
+    def pre_check(self) -> bool:
+        branches = aliases.get_build_targets(
+            *self.job_config.targets,
+        )
+        if "fedora-rawhide-x86_64" not in branches:
+            logger.debug(
+                "Skipping job configuration with no fedora-rawhide-x86_64 target."
+            )
+            return False
+        return True

packit_service/worker/events/__init__.py

@@ -50,6 +50,11 @@
 
 from packit_service.worker.events.vm_image import VMImageBuildResultEvent
 
+from packit_service.worker.events.open_scan_hub import (
+    OpenScanHubTaskFinishedEvent,
+    OpenScanHubTaskStartedEvent,
+)
+
 __all__ = [
     Event.__name__,
     EventData.__name__,
@@ -86,4 +91,6 @@
     ReleaseGitlabEvent.__name__,
     TagPushGitlabEvent.__name__,
     PullRequestFlagPagureEvent.__name__,
+    OpenScanHubTaskFinishedEvent.__name__,
+    OpenScanHubTaskStartedEvent.__name__,
 ]

packit_service/worker/events/event.py

@@ -141,6 +141,24 @@ def from_event_dict(cls, event: dict):
             branches_override=branches_override,
         )
 
+    def to_event(self) -> "Event":
+        """
+        Create an instance of Event class from the data in this class.
+        """
+        mod = __import__("packit_service.worker.events", fromlist=[self.event_type])
+        kls = getattr(mod, self.event_type)
+        kwargs = copy.copy(self.event_dict)
+        # The following data should be reconstructed by the Event instance (when needed)
+        kwargs.pop("event_type", None)
+        kwargs.pop("event_id", None)
+        kwargs.pop("task_accepted_time", None)
+        kwargs.pop("build_targets_override", None)
+        kwargs.pop("tests_targets_override", None)
+        kwargs.pop("branches_override", None)
+        pr_id = kwargs.pop("_pr_id", None)
+        kwargs["pr_id"] = pr_id
+        return kls(**kwargs)
+
     @property
     def project(self):
         if not self._project:
@@ -367,12 +385,17 @@ def store_packages_config(self):
             self.db_project_event.set_packages_config(package_config_dict)
 
     def get_non_serializable_attributes(self):
+        """List here both non serializable attributes and attributes that
+        we want to skip from the dict because are not needed to re-create
+        the event.
+        """
         return [
             "_db_project_object",
             "_db_project_event",
             "_project",
             "_base_project",
             "_package_config",
+            "_package_config_searched",
         ]
 
     def get_dict(self, default_dict: Optional[Dict] = None) -> dict:
@@ -618,10 +641,11 @@ def get_all_build_targets_by_status(
             statuses_to_filter_with=statuses_to_filter_with,
         )
 
-    def get_dict(self, default_dict: Optional[Dict] = None) -> dict:
-        result = super().get_dict()
-        result.pop("_pull_request_object")
-        return result
+    def get_non_serializable_attributes(self):
+        return super().get_non_serializable_attributes() + [
+            "fail_when_config_file_missing",
+            "_pull_request_object",
+        ]
 
 
 class AbstractResultEvent(AbstractForgeIndependentEvent):

packit_service/worker/events/open_scan_hub.py

@@ -0,0 +1,91 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+import enum
+from typing import Optional
+from logging import getLogger
+
+from ogr.abstract import GitProject
+from packit_service.config import ServiceConfig
+from packit_service.worker.events.event import AbstractResultEvent
+from packit_service.models import (
+    AbstractProjectObjectDbType,
+    ProjectEventModel,
+    OSHScanModel,
+)
+
+logger = getLogger(__name__)
+
+
+class OpenScanHubTaskAbstractEvent(AbstractResultEvent):
+
+    def __init__(
+        self,
+        task_id: int,
+        commit_sha: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.task_id = task_id
+        self.commit_sha = commit_sha
+
+        self.scan = OSHScanModel.get_by_task_id(task_id)
+        self.build = None
+        if not self.scan:
+            logger.warning(
+                f"Scan with id {task_id} not found in the database."
+                " It should have been created when receiving the CoprBuildEndEvent"
+                " and should have been associated with the copr build."
+            )
+        else:
+            self.build = self.scan.copr_build_target
+            project_event = self.build.get_project_event_model()
+            # commit_sha is needed by the StatusReporter
+            # and have to be serialized to be later found in the
+            # event metadata
+            self.commit_sha = (
+                project_event.commit_sha if not self.commit_sha else self.commit_sha
+            )
+
+    def get_db_project_object(self) -> Optional[AbstractProjectObjectDbType]:
+        return self.build.get_project_event_object()
+
+    def get_db_project_event(self) -> Optional[ProjectEventModel]:
+        return self.build.get_project_event_model()
+
+    def get_project(self) -> GitProject:
+        return ServiceConfig.get_service_config().get_project(
+            self.db_project_object.project.project_url
+        )
+
+    def get_non_serializable_attributes(self):
+        # build and scan are not serializable
+        return super().get_non_serializable_attributes() + ["build", "scan"]
+
+
+class OpenScanHubTaskFinishedEvent(OpenScanHubTaskAbstractEvent):
+
+    class Status(str, enum.Enum):
+        success = "success"
+        cancel = "cancel"
+        interrupt = "interrupt"
+        fail = "fail"
+
+    def __init__(
+        self,
+        status: Status,
+        issues_added_url: str,
+        issues_fixed_url: str,
+        scan_results_url: str,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.status = status
+        self.issues_added_url = issues_added_url
+        self.issues_fixed_url = issues_fixed_url
+        self.scan_results_url = scan_results_url
+
+
+class OpenScanHubTaskStartedEvent(OpenScanHubTaskAbstractEvent): ...