diff --git a/cycode/cli/commands/scan/code_scanner.py b/cycode/cli/commands/scan/code_scanner.py index 78ea8ca9..83da97bf 100644 --- a/cycode/cli/commands/scan/code_scanner.py +++ b/cycode/cli/commands/scan/code_scanner.py @@ -98,8 +98,21 @@ def set_issue_detected_by_scan_results(context: click.Context, scan_results: Lis set_issue_detected(context, any(scan_result.issue_detected for scan_result in scan_results)) -def _should_use_scan_service(scan_type: str, scan_parameters: Optional[dict] = None) -> bool: - return scan_type == consts.SECRET_SCAN_TYPE and scan_parameters is not None and scan_parameters['report'] is True +def _should_use_scan_service(scan_type: str, scan_parameters: dict) -> bool: + return scan_type == consts.SECRET_SCAN_TYPE and scan_parameters.get('report') is True + + +def _should_use_sync_flow(scan_type: str, sync_option: bool, scan_parameters: Optional[dict] = None) -> bool: + if not sync_option: + return False + + if scan_type not in (consts.SCA_SCAN_TYPE,): + raise ValueError(f'Sync scan is not available for {scan_type} scan type.') + + if scan_parameters.get('report') is True: + raise ValueError('You can not use sync flow with report option. Either remove "report" or "sync" option.') + + return True def _enrich_scan_result_with_data_from_detection_rules( @@ -141,6 +154,7 @@ def _get_scan_documents_thread_func( cycode_client = context.obj['client'] scan_type = context.obj['scan_type'] severity_threshold = context.obj['severity_threshold'] + sync_option = context.obj['sync'] command_scan_type = context.info_name scan_parameters['aggregation_id'] = str(_generate_unique_id()) @@ -151,7 +165,9 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local scan_id = str(_generate_unique_id()) scan_completed = False + should_use_scan_service = _should_use_scan_service(scan_type, scan_parameters) + should_use_sync_flow = _should_use_sync_flow(scan_type, sync_option, scan_parameters) try: logger.debug('Preparing local files, %s', {'batch_size': len(batch)}) @@ -166,6 +182,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local is_commit_range, scan_parameters, should_use_scan_service, + should_use_sync_flow, ) _enrich_scan_result_with_data_from_detection_rules(cycode_client, scan_type, scan_result) @@ -439,7 +456,11 @@ def perform_scan( is_commit_range: bool, scan_parameters: dict, should_use_scan_service: bool = False, + should_use_sync_flow: bool = False, ) -> ZippedFileScanResult: + if should_use_sync_flow: + return perform_scan_sync(cycode_client, zipped_documents, scan_type, scan_parameters) + if scan_type in (consts.SCA_SCAN_TYPE, consts.SAST_SCAN_TYPE) or should_use_scan_service: return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters) @@ -466,6 +487,21 @@ def perform_scan_async( ) +def perform_scan_sync( + cycode_client: 'ScanClient', + zipped_documents: 'InMemoryZip', + scan_type: str, + scan_parameters: dict, +) -> ZippedFileScanResult: + scan_results = cycode_client.zipped_file_scan_sync(zipped_documents, scan_type, scan_parameters) + logger.debug('scan request has been triggered successfully, scan id: %s', scan_results.id) + return ZippedFileScanResult( + did_detect=True, + detections_per_file=_map_detections_per_file(scan_results.detection_messages), + scan_id=scan_results.id, + ) + + def perform_commit_range_scan_async( cycode_client: 'ScanClient', from_commit_zipped_documents: 'InMemoryZip', @@ -888,10 +924,10 @@ def _map_detections_per_file(detections: List[dict]) -> List[DetectionsPerFile]: def _get_file_name_from_detection(detection: dict) -> str: - if detection['category'] == 'SAST': + if detection.get('category') == 'SAST': return detection['detection_details']['file_path'] - if detection['category'] == 'SecretDetection': + if detection.get('category') == 'SecretDetection': return _get_secret_file_name_from_detection(detection) return detection['detection_details']['file_name'] diff --git a/cycode/cli/commands/scan/scan_command.py b/cycode/cli/commands/scan/scan_command.py index a53f501b..cc97b577 100644 --- a/cycode/cli/commands/scan/scan_command.py +++ b/cycode/cli/commands/scan/scan_command.py @@ -34,7 +34,7 @@ '--scan-type', '-t', default='secret', - help='Specify the type of scan you wish to execute (the default is Secrets)', + help='Specify the type of scan you wish to execute (the default is Secrets).', type=click.Choice(config['scans']['supported_scans']), ) @click.option( @@ -100,6 +100,14 @@ type=bool, required=False, ) +@click.option( + '--sync', + is_flag=True, + default=False, + help='Run scan synchronously (the default is asynchronous).', + type=bool, + required=False, +) @click.pass_context def scan_command( context: click.Context, @@ -113,6 +121,7 @@ def scan_command( monitor: bool, report: bool, no_restore: bool, + sync: bool, ) -> int: """Scans for Secrets, IaC, SCA or SAST violations.""" if show_secret: @@ -127,6 +136,7 @@ def scan_command( context.obj['client'] = get_scan_cycode_client(client_id, secret, not context.obj['show_secret']) context.obj['scan_type'] = scan_type + context.obj['sync'] = sync context.obj['severity_threshold'] = severity_threshold context.obj['monitor'] = monitor context.obj['report'] = report diff --git a/cycode/cli/models.py b/cycode/cli/models.py index 4c6b725b..bccd4e76 100644 --- a/cycode/cli/models.py +++ b/cycode/cli/models.py @@ -2,7 +2,6 @@ from enum import Enum from typing import Dict, List, NamedTuple, Optional, Type -from cycode.cyclient import logger from cycode.cyclient.models import Detection @@ -46,8 +45,7 @@ def try_get_value(name: str) -> any: @staticmethod def get_member_weight(name: str) -> any: weight = Severity.try_get_value(name) - if weight is None: - logger.debug(f'missing severity in enum: {name}') + if weight is None: # if License Compliance return -2 return weight diff --git a/cycode/cyclient/cycode_client_base.py b/cycode/cyclient/cycode_client_base.py index a1fb68bb..6cfcd8c3 100644 --- a/cycode/cyclient/cycode_client_base.py +++ b/cycode/cyclient/cycode_client_base.py @@ -62,9 +62,14 @@ def _execute( url = self.build_full_url(self.api_url, endpoint) logger.debug(f'Executing {method.upper()} request to {url}') + timeout = self.timeout + if 'timeout' in kwargs: + timeout = kwargs['timeout'] + del kwargs['timeout'] + try: headers = self.get_request_headers(headers, without_auth=without_auth) - response = request(method=method, url=url, timeout=self.timeout, headers=headers, **kwargs) + response = request(method=method, url=url, timeout=timeout, headers=headers, **kwargs) content = 'HIDDEN' if hide_response_content_log else response.text logger.debug(f'Response {response.status_code} from {url}. Content: {content}') diff --git a/cycode/cyclient/models.py b/cycode/cyclient/models.py index 98185707..a3dff4e9 100644 --- a/cycode/cyclient/models.py +++ b/cycode/cyclient/models.py @@ -453,3 +453,21 @@ class Meta: @post_load def build_dto(self, data: Dict[str, Any], **_) -> DetectionRule: return DetectionRule(**data) + + +@dataclass +class ScanResultsSyncFlow: + id: str + detection_messages: List[Dict] + + +class ScanResultsSyncFlowSchema(Schema): + class Meta: + unknown = EXCLUDE + + id = fields.String() + detection_messages = fields.List(fields.Dict()) + + @post_load + def build_dto(self, data: Dict[str, Any], **_) -> ScanResultsSyncFlow: + return ScanResultsSyncFlow(**data) diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index 00dd8d13..c2207c9b 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -48,10 +48,20 @@ def get_detections_service_controller_path(self, scan_type: str) -> str: return self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH - def get_scan_service_url_path(self, scan_type: str, should_use_scan_service: bool = False) -> str: + @staticmethod + def get_scan_flow_type(should_use_sync_flow: bool = False) -> str: + if should_use_sync_flow: + return '/sync' + + return '' + + def get_scan_service_url_path( + self, scan_type: str, should_use_scan_service: bool = False, should_use_sync_flow: bool = False + ) -> str: service_path = self.scan_config.get_service_name(scan_type, should_use_scan_service) controller_path = self.get_scan_controller_path(scan_type) - return f'{service_path}/{controller_path}' + flow_type = self.get_scan_flow_type(should_use_sync_flow) + return f'{service_path}/{controller_path}{flow_type}' def content_scan(self, scan_type: str, file_name: str, content: str, is_git_diff: bool = True) -> models.ScanResult: path = f'{self.get_scan_service_url_path(scan_type)}/content' @@ -82,12 +92,28 @@ def get_scan_report_url(self, scan_id: str, scan_type: str) -> models.ScanReport response = self.scan_cycode_client.get(url_path=self.get_scan_report_url_path(scan_id, scan_type)) return models.ScanReportUrlResponseSchema().build_dto(response.json()) - def get_zipped_file_scan_async_url_path(self, scan_type: str) -> str: + def get_zipped_file_scan_async_url_path(self, scan_type: str, should_use_sync_flow: bool = False) -> str: async_scan_type = self.scan_config.get_async_scan_type(scan_type) async_entity_type = self.scan_config.get_async_entity_type(scan_type) - scan_service_url_path = self.get_scan_service_url_path(scan_type, True) + scan_service_url_path = self.get_scan_service_url_path( + scan_type, should_use_scan_service=True, should_use_sync_flow=should_use_sync_flow + ) return f'{scan_service_url_path}/{async_scan_type}/{async_entity_type}' + def zipped_file_scan_sync( + self, zip_file: InMemoryZip, scan_type: str, scan_parameters: dict + ) -> models.ScanResultsSyncFlow: + files = {'file': ('multiple_files_scan.zip', zip_file.read())} + del scan_parameters['report'] # BE raises validation error instead of ignoring it + response = self.scan_cycode_client.post( + url_path=self.get_zipped_file_scan_async_url_path(scan_type, should_use_sync_flow=True), + data={'scan_parameters': json.dumps(scan_parameters)}, + files=files, + hide_response_content_log=self._hide_response_log, + timeout=60, + ) + return models.ScanResultsSyncFlowSchema().load(response.json()) + def zipped_file_scan_async( self, zip_file: InMemoryZip,