From 4aa54f0b5a8ddc1d18b94b13018fcb9b25e5d4b9 Mon Sep 17 00:00:00 2001 From: zkdev Date: Thu, 25 Apr 2024 15:05:19 +0200 Subject: [PATCH] Introduce finding-type specific issue-replication cfg Useful to toggle issue-replication per finding-type without code changes. --- config.py | 67 +++++++++++++++++------- issue_replicator/__main__.py | 98 ++++++++++++++++++++++-------------- issue_replicator/github.py | 3 +- 3 files changed, 111 insertions(+), 57 deletions(-) diff --git a/config.py b/config.py index 8f6ba857..d7e10ab7 100644 --- a/config.py +++ b/config.py @@ -12,6 +12,7 @@ import concourse.model.traits.image_scan as image_scan import cnudie.iter import dso.cvss +import dso.model import gci.componentmodel as cm import github.compliance.model import protecode.model @@ -136,6 +137,27 @@ class BDBAConfig: delete_inactive_products_after_seconds: int +@dataclasses.dataclass(frozen=True) +class FindingTypeIssueReplicationCfgBase: + ''' + :param str finding_type: + finding type this configuration should be applied for + (see cc-utils dso/model.py for available "Datatype"s) + :param bool enable_issue_assignees + ''' + finding_type: str + enable_issue_assignees: bool + + +@dataclasses.dataclass(frozen=True) +class VulnerabilityIssueReplicationCfg(FindingTypeIssueReplicationCfgBase): + ''' + :param int cve_threshold: + vulnerability findings below this threshold won't be reported in the issue(s) + ''' + cve_threshold: int + + @dataclasses.dataclass(frozen=True) class IssueReplicatorConfig: ''' @@ -145,8 +167,6 @@ class IssueReplicatorConfig: time after which an issue must be updated at latest :param int lookup_new_backlog_item_interval: time to wait in case no backlog item was found before searching for new backlog item again - :param int cve_threshold: - vulnerability findings below this threshold won't be reported in the issue(s) :param LicenseCfg license_cfg: required to differentiate between allowed and prohibited licenses :param MaxProcessingTimesDays max_processing_days: @@ -159,19 +179,19 @@ class IssueReplicatorConfig: labels matching one of these regexes won't be removed upon an issue update :param int number_included_closed_issues: number of closed issues to consider when evaluating creating vs re-opening an issue - :param bool enable_issue_assignees :param tuple[str] artefact_types: list of artefact types for which issues should be created, other artefact types are skipped :param Callable[Node, bool] node_filter: filter of artefact nodes to explicitly in- or exclude artefacts from the issue replication :param tuple[RescoringRule] cve_rescoring_rules: these rules are applied to calculate proposed rescorings which are displayed in the issue + :param tuple[FindingTypeIssueReplicationCfgBase] finding_type_issue_replication_cfgs: + these cfgs are finding type specific and allow fine granular configuration ''' delivery_service_url: str delivery_dashboard_url: str replication_interval: int lookup_new_backlog_item_interval: int - cve_threshold: int license_cfg: image_scan.LicenseCfg max_processing_days: github.compliance.model.MaxProcessingTimesDays github_api_lookup: typing.Callable[[str], github3.GitHub] @@ -179,10 +199,10 @@ class IssueReplicatorConfig: github_issue_template_cfgs: tuple[image_scan.GithubIssueTemplateCfg] github_issue_labels_to_preserve: set[str] number_included_closed_issues: int - enable_issue_assignees: bool artefact_types: tuple[str] node_filter: typing.Callable[[cnudie.iter.Node], bool] cve_rescoring_rules: tuple[dso.cvss.RescoringRule] + finding_type_issue_replication_cfgs: tuple[FindingTypeIssueReplicationCfgBase] @dataclasses.dataclass(frozen=True) @@ -554,11 +574,6 @@ def deserialise_issue_replicator_config( default_value=60, ) - cve_threshold = deserialise_config_property( - config=issue_replicator_config, - property_key='cve_threshold', - ) - prohibited_licenses = deserialise_config_property( config=issue_replicator_config, property_key='prohibited_licenses', @@ -610,12 +625,6 @@ def deserialise_issue_replicator_config( default_value=0, ) - enable_issue_assignees = deserialise_config_property( - config=issue_replicator_config, - property_key='enable_issue_assignees', - default_value=True, - ) - artefact_types = tuple(deserialise_config_property( config=issue_replicator_config, property_key='artefact_types', @@ -647,12 +656,34 @@ def deserialise_issue_replicator_config( ) cve_rescoring_rules = tuple(dso.cvss.rescoring_rules_from_dicts(cve_rescoring_rules_raw)) + finding_type_issue_replication_cfgs_raw = deserialise_config_property( + config=issue_replicator_config, + property_key='finding_type_issue_replication_configs', + default_config=default_config, + default_value=[], + ) + + model_class_for_finding_type = { + dso.model.Datatype.VULNERABILITY: VulnerabilityIssueReplicationCfg, + dso.model.Datatype.LICENSE: FindingTypeIssueReplicationCfgBase, + } + + finding_type_issue_replication_cfgs = tuple( + dacite.from_dict( + data_class=model_class_for_finding_type.get( + finding_type_issue_replication_cfg_raw['finding_type'], + FindingTypeIssueReplicationCfgBase, + ), + data=finding_type_issue_replication_cfg_raw, + ) + for finding_type_issue_replication_cfg_raw in finding_type_issue_replication_cfgs_raw + ) + return IssueReplicatorConfig( delivery_service_url=delivery_service_url, delivery_dashboard_url=delivery_dashboard_url, replication_interval=replication_interval, lookup_new_backlog_item_interval=lookup_new_backlog_item_interval, - cve_threshold=cve_threshold, license_cfg=license_cfg, max_processing_days=max_processing_days, github_api_lookup=github_api_lookup, @@ -660,10 +691,10 @@ def deserialise_issue_replicator_config( github_issue_template_cfgs=github_issue_template_cfgs, github_issue_labels_to_preserve=github_issue_labels_to_preserve, number_included_closed_issues=number_included_closed_issues, - enable_issue_assignees=enable_issue_assignees, artefact_types=artefact_types, node_filter=node_filter, cve_rescoring_rules=cve_rescoring_rules, + finding_type_issue_replication_cfgs=finding_type_issue_replication_cfgs, ) diff --git a/issue_replicator/__main__.py b/issue_replicator/__main__.py index dc0e6113..e1d40ec7 100644 --- a/issue_replicator/__main__.py +++ b/issue_replicator/__main__.py @@ -283,10 +283,10 @@ def _findings_by_type_and_date( None, ]: ''' - yields all findings of the given `artefact` in `components`, grouped by finding type and latest - processing date. Also, it yields the information whether the artefact was scanned at all which - is determined based on if there is a "dummy finding". Thresholds provided by configuration are - applied on the findings before yielding. + yields all findings (of configured types) of the given `artefact` in `components`, grouped by + finding type and latest processing date. Also, it yields the information whether the artefact + was scanned at all which is determined based on if there is a "dummy finding". Thresholds + provided by configuration are applied on the findings before yielding. ''' findings = tuple(_iter_findings_for_artefact( delivery_client=delivery_client, @@ -297,44 +297,46 @@ def _findings_by_type_and_date( sprints = sprint_dates(delivery_client=delivery_client) + datasource_for_datatype = { + dso.model.Datatype.VULNERABILITY: dso.model.Datasource.BDBA, + dso.model.Datatype.LICENSE: dso.model.Datasource.BDBA, + dso.model.Datatype.MALWARE: dso.model.Datasource.CLAMAV, + } + for latest_processing_date in latest_processing_dates: date = dateutil.parser.isoparse(latest_processing_date).date() - vulnerability_bdba_findings, is_scanned = _findings_for_type_and_date( - issue_replicator_config=issue_replicator_config, - latest_processing_date=date, - sprints=sprints, - type=dso.model.Datatype.VULNERABILITY, - source=dso.model.Datasource.BDBA, - findings=findings, - ) - vulnerability_bdba_findings = tuple( - finding for finding in vulnerability_bdba_findings - if finding.finding.data.cvss_v3_score >= issue_replicator_config.cve_threshold - ) - yield ( - dso.model.Datatype.VULNERABILITY, - dso.model.Datasource.BDBA, - date, - vulnerability_bdba_findings, - is_scanned, - ) + for finding_type_cfg in issue_replicator_config.finding_type_issue_replication_cfgs: + finding_type = finding_type_cfg.finding_type + finding_source = datasource_for_datatype.get(finding_type) - license_bdba_findings, is_scanned = _findings_for_type_and_date( - issue_replicator_config=issue_replicator_config, - latest_processing_date=date, - sprints=sprints, - type=dso.model.Datatype.LICENSE, - source=dso.model.Datasource.BDBA, - findings=findings, - ) - yield ( - dso.model.Datatype.LICENSE, - dso.model.Datasource.BDBA, - date, - license_bdba_findings, - is_scanned, - ) + logger.info(f'processing findings of {finding_type=} with {finding_type_cfg=}') + + findings, is_scanned = _findings_for_type_and_date( + issue_replicator_config=issue_replicator_config, + latest_processing_date=date, + sprints=sprints, + type=finding_type, + source=finding_source, + findings=findings, + ) + + if ( + finding_type == dso.model.Datatype.VULNERABILITY + and isinstance(finding_type_cfg, config.VulnerabilityIssueReplicationCfg) + ): + findings = tuple( + finding for finding in findings + if finding.finding.data.cvss_v3_score >= finding_type_cfg.cve_threshold + ) + + yield ( + finding_type, + finding_source, + date, + findings, + is_scanned, + ) def replicate_issue( @@ -413,13 +415,33 @@ def replicate_issue( latest_processing_dates=correlation_ids_by_latest_processing_date.keys(), ) + def _find_finding_type_issue_replication_cfg( + finding_cfgs: collections.abc.Iterable[config.FindingTypeIssueReplicationCfgBase], + finding_type: str, + absent_ok: bool=False, + ) -> config.FindingTypeIssueReplicationCfgBase: + for finding_cfg in finding_cfgs: + if finding_cfg.finding_type == finding_type: + return finding_cfg + + if absent_ok: + return None + + return ValueError(f'no finding-type specific cfg found for {finding_type=}') + is_in_bom = len(active_compliance_snapshots_for_artefact) > 0 for type, source, date, findings, is_scanned in findings_by_type_and_date: correlation_id = correlation_ids_by_latest_processing_date.get(date.isoformat()) + finding_type_issue_replication_cfg = _find_finding_type_issue_replication_cfg( + finding_cfgs=issue_replicator_config.finding_type_issue_replication_cfgs, + finding_type=type, + ) + issue_replicator.github.create_or_update_or_close_issue( cfg_name=cfg_name, issue_replicator_config=issue_replicator_config, + finding_type_issue_replication_cfg=finding_type_issue_replication_cfg, delivery_client=delivery_client, type=type, source=source, diff --git a/issue_replicator/github.py b/issue_replicator/github.py index 835f1995..8d9087f7 100644 --- a/issue_replicator/github.py +++ b/issue_replicator/github.py @@ -679,6 +679,7 @@ def update_issue( def create_or_update_or_close_issue( cfg_name: str, issue_replicator_config: config.IssueReplicatorConfig, + finding_type_issue_replication_cfg: config.FindingTypeIssueReplicationCfgBase, delivery_client: delivery.client.DeliveryServiceClient, type: str, source: str, @@ -763,7 +764,7 @@ def labels_to_preserve( # not scanned yet but no open issue found either -> nothing to do return - if issue_replicator_config.enable_issue_assignees: + if finding_type_issue_replication_cfg.enable_issue_assignees: assignees, assignees_statuses = _issue_assignees( issue_replicator_config=issue_replicator_config, delivery_client=delivery_client,