From 0763774803f5c85f5691dac885301202f7f0b094 Mon Sep 17 00:00:00 2001 From: I745280 Date: Mon, 19 Feb 2024 16:05:15 +0100 Subject: [PATCH 01/30] Remove old SAST folder --- SAST/__init__.py | 0 SAST/codeql/Dockerfile | 23 ---- SAST/codeql/__init__.py | 0 SAST/codeql/codeql-install.sh | 45 -------- SAST/codeql/codeql-versions-list.yaml | 7 -- SAST/codeql/codeql_v2_13_1/.gitignore | 1 - SAST/codeql/codeql_v2_13_1/__init__.py | 0 SAST/codeql/codeql_v2_13_1/codeql.py | 19 ---- SAST/codeql/codeql_v2_13_1/config.yaml | 17 --- SAST/codeql/codeql_v2_9_2/.gitignore | 1 - SAST/codeql/codeql_v2_9_2/__init__.py | 0 SAST/codeql/codeql_v2_9_2/codeql.py | 22 ---- SAST/codeql/codeql_v2_9_2/config.yaml | 17 --- SAST/codeql/core/__init__.py | 0 SAST/codeql/core/codeql.py | 103 ------------------ SAST/codeql/core/resources/_template_build.sh | 3 - SAST/requirements.txt | 7 -- SAST/sast-config.yaml | 32 ------ 18 files changed, 297 deletions(-) delete mode 100644 SAST/__init__.py delete mode 100644 SAST/codeql/Dockerfile delete mode 100644 SAST/codeql/__init__.py delete mode 100755 SAST/codeql/codeql-install.sh delete mode 100644 SAST/codeql/codeql-versions-list.yaml delete mode 100644 SAST/codeql/codeql_v2_13_1/.gitignore delete mode 100644 SAST/codeql/codeql_v2_13_1/__init__.py delete mode 100644 SAST/codeql/codeql_v2_13_1/codeql.py delete mode 100644 SAST/codeql/codeql_v2_13_1/config.yaml delete mode 100644 SAST/codeql/codeql_v2_9_2/.gitignore delete mode 100644 SAST/codeql/codeql_v2_9_2/__init__.py delete mode 100644 SAST/codeql/codeql_v2_9_2/codeql.py delete mode 100644 SAST/codeql/codeql_v2_9_2/config.yaml delete mode 100644 SAST/codeql/core/__init__.py delete mode 100644 SAST/codeql/core/codeql.py delete mode 100644 SAST/codeql/core/resources/_template_build.sh delete mode 100644 SAST/requirements.txt delete mode 100644 SAST/sast-config.yaml diff --git a/SAST/__init__.py b/SAST/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/SAST/codeql/Dockerfile b/SAST/codeql/Dockerfile deleted file mode 100644 index 2dd7542..0000000 --- a/SAST/codeql/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -################################################################################ -# base system -################################################################################ - -FROM adoptopenjdk/openjdk11 - -ARG TPF_HOME="/tp-framework" -ARG CODEQL_INTERFACE_DIR="${TPF_HOME}/SAST/codeql" - -WORKDIR ${CODEQL_INTERFACE_DIR} - -# copy necessary files for codeql -COPY codeql_v2_9_2/ ${CODEQL_INTERFACE_DIR}/codeql_v2_9_2/ -COPY codeql_v2_13_1/ ${CODEQL_INTERFACE_DIR}/codeql_v2_13_1/ -COPY core/ ${CODEQL_INTERFACE_DIR}/core/ -COPY ./codeql-install.sh ${CODEQL_INTERFACE_DIR} -COPY ./codeql-versions-list.yaml ${CODEQL_INTERFACE_DIR} - -RUN apt-get update -RUN apt-get install wget - -# codeql installation -RUN ${CODEQL_INTERFACE_DIR}/codeql-install.sh ${CODEQL_INTERFACE_DIR}/codeql-versions-list.yaml \ No newline at end of file diff --git a/SAST/codeql/__init__.py b/SAST/codeql/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/SAST/codeql/codeql-install.sh b/SAST/codeql/codeql-install.sh deleted file mode 100755 index b1089a3..0000000 --- a/SAST/codeql/codeql-install.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -VERSION_LIST_FILE=$1 -CODEQL_DIR=/codeql - -mkdir $CODEQL_DIR - -# Make sure the file contains a newline at the end -echo -en '\n' >> "$VERSION_LIST_FILE" - -name="" -link="" -# Read the VERSION_LIST_FILE line by line -while IFS=: read -r key value; do - # Remove leading/trailing spaces from the key and value (remove " as well) - key=$(echo "$key" | xargs) - value=$(echo "$value" | xargs | tr -d '\"') - - # Check if the line contains "link" or "name" field - if [[ "$key" == "link" ]]; then - link=$value - elif [[ "$key" == "name" ]]; then - name=$value - fi - - # Check if both name and link are set - if [[ -n "$name" && -n "$link" ]]; then - # Download the codeql version using wget, extract it and move it to the codeql directory - echo "Downloading $name from $link" - wget -q "$link" - tar -xzf ./codeql-bundle-linux64.tar.gz -C "$CODEQL_DIR" - mv "$CODEQL_DIR/codeql" "$CODEQL_DIR/$name" - - # setup codeql version - "$CODEQL_DIR/$name/codeql" resolve languages - "$CODEQL_DIR/$name/codeql" resolve qlpacks - - # remove the downloaded .tar.gz - rm ./codeql-bundle-linux64.tar.gz - - # Reset the variables for the next entry - name="" - link="" - fi -done < "$VERSION_LIST_FILE" diff --git a/SAST/codeql/codeql-versions-list.yaml b/SAST/codeql/codeql-versions-list.yaml deleted file mode 100644 index aba131a..0000000 --- a/SAST/codeql/codeql-versions-list.yaml +++ /dev/null @@ -1,7 +0,0 @@ -versions: - # 2_9_2: - # name: "codeql_v2_9_2" - # link: "https://github.com/github/codeql-action/releases/download/codeql-bundle-20220512/codeql-bundle-linux64.tar.gz" - 2_13_1: - name: "codeql_v2_13_1" - link: "https://github.com/github/codeql-action/releases/download/codeql-bundle-20230428/codeql-bundle-linux64.tar.gz" diff --git a/SAST/codeql/codeql_v2_13_1/.gitignore b/SAST/codeql/codeql_v2_13_1/.gitignore deleted file mode 100644 index d479fbf..0000000 --- a/SAST/codeql/codeql_v2_13_1/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tmp_res/ \ No newline at end of file diff --git a/SAST/codeql/codeql_v2_13_1/__init__.py b/SAST/codeql/codeql_v2_13_1/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/SAST/codeql/codeql_v2_13_1/codeql.py b/SAST/codeql/codeql_v2_13_1/codeql.py deleted file mode 100644 index ce8b4ac..0000000 --- a/SAST/codeql/codeql_v2_13_1/codeql.py +++ /dev/null @@ -1,19 +0,0 @@ -from pathlib import Path -import yaml -from typing import Dict -import sys - -DIR: Path = Path(__file__).parent.resolve() -SAST_DIR: Path = DIR.parent.parent.resolve() -sys.path.append(str(SAST_DIR)) - -from codeql.core.codeql import CodeQL - -class CodeQL_v_2_13_1(CodeQL): - - def __init__(self): - self.tool = "codeql_2_13_1" - self.CODEQL_SCRIPT_DIR: Path = Path(__file__).parent.resolve() - self.CODEQL_CONFIG_FILE: Path = self.CODEQL_SCRIPT_DIR / "config.yaml" - with open(self.CODEQL_CONFIG_FILE) as sast_config_file: - self.CODEQL_CONFIG: Dict = yaml.load(sast_config_file, Loader=yaml.Loader) diff --git a/SAST/codeql/codeql_v2_13_1/config.yaml b/SAST/codeql/codeql_v2_13_1/config.yaml deleted file mode 100644 index 74dd344..0000000 --- a/SAST/codeql/codeql_v2_13_1/config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Mandatory configuration parameters -name: "codeql" -version: "2.13.1" -supported_languages: - - "JS" - - "JAVA" - - "PYTHON" - - "RUBY" - - "CPP" - - "CSHARP" - - "GO" -tool_interface: "SAST.codeql.codeql_v2_13_1.codeql.CodeQL_v_2_13_1" -supported_vulnerability: - xss: "xss" - -# Specific SAST config parameters -installation_path: "/codeql/codeql_v2_13_1" diff --git a/SAST/codeql/codeql_v2_9_2/.gitignore b/SAST/codeql/codeql_v2_9_2/.gitignore deleted file mode 100644 index d479fbf..0000000 --- a/SAST/codeql/codeql_v2_9_2/.gitignore +++ /dev/null @@ -1 +0,0 @@ -tmp_res/ \ No newline at end of file diff --git a/SAST/codeql/codeql_v2_9_2/__init__.py b/SAST/codeql/codeql_v2_9_2/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/SAST/codeql/codeql_v2_9_2/codeql.py b/SAST/codeql/codeql_v2_9_2/codeql.py deleted file mode 100644 index aa44633..0000000 --- a/SAST/codeql/codeql_v2_9_2/codeql.py +++ /dev/null @@ -1,22 +0,0 @@ -from pathlib import Path -import yaml -from typing import Dict -import sys - -DIR: Path = Path(__file__).parent.resolve() -SAST_DIR: Path = DIR.parent.parent.resolve() -sys.path.append(str(SAST_DIR)) - -from codeql.core.codeql import CodeQL - -class CodeQL_v_2_9_2(CodeQL): - - def __init__(self): - self.tool = "codeql_2_9_2" - self.CODEQL_SCRIPT_DIR: Path = Path(__file__).parent.resolve() - self.CODEQL_CONFIG_FILE: Path = self.CODEQL_SCRIPT_DIR / "config.yaml" - with open(self.CODEQL_CONFIG_FILE) as sast_config_file: - self.CODEQL_CONFIG: Dict = yaml.load(sast_config_file, Loader=yaml.Loader) - - - diff --git a/SAST/codeql/codeql_v2_9_2/config.yaml b/SAST/codeql/codeql_v2_9_2/config.yaml deleted file mode 100644 index 96f26c2..0000000 --- a/SAST/codeql/codeql_v2_9_2/config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Mandatory configuration parameters -name: "codeql" -version: "2.9.2" -supported_languages: - - "JS" - - "JAVA" - - "PYTHON" - - "RUBY" - - "CPP" - - "CSHARP" - - "GO" -tool_interface: "SAST.codeql.codeql_v2_9_2.codeql.CodeQL_v_2_9_2" -supported_vulnerability: - xss: "xss" - -# Specific SAST config parameters -installation_path: "/codeql/codeql_v2_9_2" diff --git a/SAST/codeql/core/__init__.py b/SAST/codeql/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/SAST/codeql/core/codeql.py b/SAST/codeql/core/codeql.py deleted file mode 100644 index 279ff47..0000000 --- a/SAST/codeql/core/codeql.py +++ /dev/null @@ -1,103 +0,0 @@ -import asyncio -import json -import shutil -from pathlib import Path -from typing import Dict -import yaml - -import logging -from core import loggermgr -logger = logging.getLogger(loggermgr.logger_name(__name__)) - -from core.sast import SAST -import config - -SAST_ROOT_DIR: Path = config.ROOT_SAST_DIR -CODEQL_BUILD_TEMPLATE: Path = SAST_ROOT_DIR / "core/resources/_template_build.sh" - -class CodeQL(SAST): - tool = "codeql" - - CODEQL_SCRIPT_DIR: Path = None - CODEQL_CONFIG_FILE: Path = None - CODEQL_CONFIG: Dict = None - - async def launcher(self, src_dir: Path, language: str, - output_dir: Path, **kwargs) -> Path: - self.logging(what="launcher", status="started...") - # project_name: str = f"TPF_{language}_{src_dir.name}_{uuid.uuid4()}" - project_name: str = SAST.build_project_name(src_dir.name, self.tool, language, timestamp=True) - proj_dir_tmp: Path = output_dir / project_name - proj_dir_tmp.mkdir(parents=True, exist_ok=True) - shutil.copytree(src_dir, proj_dir_tmp / "src") - src_root: Path = (proj_dir_tmp / "src").resolve() - codeql_db_location: Path = (src_root / "ql_db").resolve() - - language = language.lower() - if language == "js": - language = "javascript" - - # Preparing the building command - self.logging(what="launcher", message=f"building", status="started...") - codeql_createdb_cmd = f"{self.CODEQL_CONFIG['installation_path']}/codeql database create {codeql_db_location} --source-root {src_root} --language {language}" - if "lib_dir" in kwargs and kwargs["lib_dir"]: - lib_dir: Path = Path(src_root / kwargs["lib_dir"]).resolve() - # Read the build template command file and prepare it - with open(CODEQL_BUILD_TEMPLATE, 'r') as build_template_file: - build_filedata = build_template_file.read() - build_filedata = build_filedata.replace("$PATTERNSRC", f"{src_root}/src") - build_filedata = build_filedata.replace("$PATTERNLIB", f"{lib_dir}") - build_filedata = build_filedata.replace("$TPFOUT", f"{proj_dir_tmp}") - # Write the build command file - with open(f"{proj_dir_tmp}/build.sh", 'w', 0o777) as build_file: - build_file.write(build_filedata) - pattern_build_cmd = f"\'{proj_dir_tmp}/build.sh\'" - codeql_createdb_cmd += " --command={}".format(pattern_build_cmd) - # if kwargs["measurement"]: - # pattern_build_cmd = f"\'find {src_root} -type f -name \"*.java\" -exec javac -cp ../lib/*.jar\'" - # codeql_createdb_cmd = f"{self.CODEQL_CONFIG['installation_path']}/codeql database create {codeql_db_location} --source-root {src_root} --language {language} --command={pattern_build_cmd}" - - codeql_createdb = await asyncio.create_subprocess_shell(codeql_createdb_cmd) - await codeql_createdb.wait() - self.logging(what="launcher", message=f"building", status="done.") - - self.logging(what="launcher", message=f"scanning", status="started...") - codeql_analyze_cmd = f"{self.CODEQL_CONFIG['installation_path']}/codeql database analyze {codeql_db_location} --format=sarif-latest --output={proj_dir_tmp}/{project_name}.sarif" - codeql_analyze = await asyncio.create_subprocess_shell(codeql_analyze_cmd) - await codeql_analyze.wait() - self.logging(what="launcher", message=f"scanning", status="done.") - - res = proj_dir_tmp / f"{project_name}.sarif" - self.logging(what="launcher", message=f"result {res}", status="done.") - return res - - - def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: - self.logging(what="inspector", status="started...") - with open(sast_res_file) as sarif_file: - codeql_sarif_report: Dict = json.load(sarif_file) - - sarif_results: list[Dict] = codeql_sarif_report["runs"][0]["results"] - supported_vulnerabilities = list(self.CODEQL_CONFIG["supported_vulnerability"].values()) - - # TODO - SAST: are we properly mapping the vulns here? - sarif_results = list( - filter(lambda x: [SAST.vuln_match(vuln, x["ruleId"]) for vuln in supported_vulnerabilities], sarif_results) - ) - - findings: list[Dict] = [] - for sarif_res in sarif_results: - for location in sarif_res["locations"]: - finding: Dict = { - "type": SAST.get_norm_vuln(sarif_res['ruleId'], self.CODEQL_CONFIG["supported_vulnerability"]), - "type_orig": sarif_res['ruleId'], - "file": location["physicalLocation"]["artifactLocation"]["uri"].split("/")[-1], - "line": location["physicalLocation"]["region"]["startLine"] - } - findings.append(finding) - self.logging(what="inspector", status="done.") - return findings - - - async def get_tool_version(self) -> str: - return self.CODEQL_CONFIG["version"] \ No newline at end of file diff --git a/SAST/codeql/core/resources/_template_build.sh b/SAST/codeql/core/resources/_template_build.sh deleted file mode 100644 index 51275f5..0000000 --- a/SAST/codeql/core/resources/_template_build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -find $PATTERNSRC -name "*.java" > $TPFOUT/sources.txt -javac -classpath $PATTERNLIB/*.jar @$TPFOUT/sources.txt \ No newline at end of file diff --git a/SAST/requirements.txt b/SAST/requirements.txt deleted file mode 100644 index 628834f..0000000 --- a/SAST/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -####################################################################### -## WARNING: -## if the SAST tools make use of some requirements.txt files, do not -## use them directly as they mess up with our docker environment. Just -## copy the content of those files hereafter -####################################################################### -PyYAML==6.0 \ No newline at end of file diff --git a/SAST/sast-config.yaml b/SAST/sast-config.yaml deleted file mode 100644 index e317237..0000000 --- a/SAST/sast-config.yaml +++ /dev/null @@ -1,32 +0,0 @@ -############################################################### -# SAST tools config -############################################################### -tools: - codeql: - version: - # 2.9.2: - # config: "./codeql/codeql_v2_9_2/config.yaml" - # deploy: true - 2.13.1: - config: "./codeql/codeql_v2_13_1/config.yaml" - deploy: true - -############################################################### -# Programming language config -# - The following are intended as guidelines for SAST tools when -# preparing their scans -# - However, SAST tools may override the following configurations -# in their own interface implementations -# - For instance, a SAST tool may decide that it does not need -# to compile Java projects for their scan -############################################################### -language: - php: - sast_compile: false - compile_default_instruction: "php -d opcache.enable_cli=1 -d opcache.opt_debug_level=0x10000 " - java: - sast_compile: true - compile_default_instruction: "javac -cp " - js: - sast_compile: false - compile_default_instruction: null From f655669b0e16f58d8a015b6458ba900c8c31e07d Mon Sep 17 00:00:00 2001 From: I745280 Date: Mon, 19 Feb 2024 16:09:51 +0100 Subject: [PATCH 02/30] Add SAST Submodule. --- .gitmodules | 4 ++++ SAST | 1 + 2 files changed, 5 insertions(+) create mode 160000 SAST diff --git a/.gitmodules b/.gitmodules index c100b58..6bbed40 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,7 @@ [submodule "testability_patterns"] path = testability_patterns url = git@github.com:testable-eu/sast-testability-patterns.git +[submodule "SAST"] + path = SAST + url = https://github.com/sacumesh/SAST + branch = feature/sast-extraction diff --git a/SAST b/SAST new file mode 160000 index 0000000..98cfd29 --- /dev/null +++ b/SAST @@ -0,0 +1 @@ +Subproject commit 98cfd29f7c80a824042c318f94a170d89fd00d23 From dedf632180f46829da68b035b0f54403114f26ff Mon Sep 17 00:00:00 2001 From: I745280 Date: Mon, 19 Feb 2024 16:13:49 +0100 Subject: [PATCH 03/30] Remove old SAST interface module --- qualitytests/core/sast_test.py | 2 +- tp_framework/core/sast.py | 80 ---------------------------------- 2 files changed, 1 insertion(+), 81 deletions(-) delete mode 100644 tp_framework/core/sast.py diff --git a/qualitytests/core/sast_test.py b/qualitytests/core/sast_test.py index 3000b3d..4de3914 100644 --- a/qualitytests/core/sast_test.py +++ b/qualitytests/core/sast_test.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Dict -from tp_framework.core.sast import SAST +from SAST.src.sast import SAST class SastTest(SAST): diff --git a/tp_framework/core/sast.py b/tp_framework/core/sast.py deleted file mode 100644 index 62226d9..0000000 --- a/tp_framework/core/sast.py +++ /dev/null @@ -1,80 +0,0 @@ -import abc -from pathlib import Path -from typing import Dict -from datetime import datetime - -import logging -from core import loggermgr -logger = logging.getLogger(loggermgr.logger_name(__name__)) - -from core import utils - - -class SAST(metaclass=abc.ABCMeta): - tool = None - - @classmethod - def __subclasshook__(cls, subclass): - return (hasattr(subclass, "launcher") and callable(subclass.launcher) and - hasattr(subclass, "inspector") and callable(subclass.inspector) and - hasattr(subclass, "get_tool_version") and callable(subclass.get_tool_version) or - NotImplemented) - - - @abc.abstractmethod - async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: - raise NotImplementedError - - - @abc.abstractmethod - def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: - """ - Shall return a list of Dict s.t. : - [{ - type: "", - file: "", - line: "" - }, ...] - """ - raise NotImplementedError - - - @abc.abstractmethod - async def get_tool_version(self) -> str: - raise NotImplementedError - - - @staticmethod - def build_project_name(name: str, tool: str | None, language: str, timestamp: bool = True): - now = None - if timestamp: - now = datetime.now() - if tool: - comp = f"{tool}_{name}" - else: - comp = name - return utils.build_timestamp_language_name(comp, language, now, extra="TPF") - - - def logging(self, what="launcher", message=None, status=None): - messagestr = "" - if message: - messagestr = f" - {message}" - statusstr = "" - if status: - statusstr = f": {status}" - logger.info(f"SAST tool {self.tool} - {what}{messagestr}{statusstr}") - - - @staticmethod - def get_norm_vuln(vuln: str, d_supported_vuln_map: Dict): - for norm_vuln in d_supported_vuln_map: - if SAST.vuln_match(d_supported_vuln_map[norm_vuln], vuln): - return norm_vuln - return None - - - @staticmethod - # simple sub-string but it could be elaborated more - def vuln_match(vcand, vtarget): - return vcand in vtarget From 7a886ba5d19e4fc45b58ad2ae062bb7242b41263 Mon Sep 17 00:00:00 2001 From: I745280 Date: Wed, 21 Feb 2024 14:22:37 +0100 Subject: [PATCH 04/30] Renamed SAST to sast --- .gitmodules | 9 +++++---- SAST | 1 - sast | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) delete mode 160000 SAST create mode 160000 sast diff --git a/.gitmodules b/.gitmodules index 6bbed40..22d5a1f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,8 @@ [submodule "testability_patterns"] path = testability_patterns url = git@github.com:testable-eu/sast-testability-patterns.git -[submodule "SAST"] - path = SAST - url = https://github.com/sacumesh/SAST - branch = feature/sast-extraction +[submodule "sast"] + path = sast + url = git@github.com:sacumesh/sast.git + branch = feature/sast-extraction + diff --git a/SAST b/SAST deleted file mode 160000 index 98cfd29..0000000 --- a/SAST +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 98cfd29f7c80a824042c318f94a170d89fd00d23 diff --git a/sast b/sast new file mode 160000 index 0000000..4502bf6 --- /dev/null +++ b/sast @@ -0,0 +1 @@ +Subproject commit 4502bf671967b0fa716174a9a7822efa15728f9c From 45ccb288f086f126b3a56ac0b153f33326336fdb Mon Sep 17 00:00:00 2001 From: I745280 Date: Thu, 22 Feb 2024 01:10:32 +0100 Subject: [PATCH 05/30] Integrate sast tools middle --- Dockerfile | 6 ++++-- docker-compose-dev.yml | 4 ++-- requirements-dev.txt | Bin 162 -> 454 bytes tp_framework/core/analysis.py | 19 +++++-------------- tp_framework/core/discovery.py | 3 ++- tp_framework/core/measure.py | 4 +++- tp_framework/core/modelling_rules.py | 9 ++------- 7 files changed, 18 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 552ec3f..9583908 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,10 @@ COPY testability_patterns ${TPF_HOME}/testability_patterns COPY config.py ${TPF_HOME}/config.py COPY setup.py ${TPF_HOME}/setup.py COPY pytest.ini ${TPF_HOME}/pytest.ini -COPY SAST/requirements.txt ${SAST_DIR}/requirements.txt -COPY SAST/sast-config.yaml ${SAST_DIR}/sast-config.yaml +COPY sast/requirements.txt ${SAST_DIR}/requirements.txt +COPY sast/sast/sast-config.yaml ${SAST_DIR}/sast-config.yaml + +COPY sast ${TPF_HOME}/sast ARG TESTS_DIR="qualitytests" COPY ${TESTS_DIR}/requirements.txt ${TPF_HOME}/${TESTS_DIR}/requirements.txt diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index dfefacb..c071ceb 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -5,7 +5,7 @@ services: codeql: image: tpf_codeql build: - context: './SAST/codeql' + context: './sast/sast/codeql' dockerfile: "./Dockerfile" volumes: - codeql_interface:/tp-framework/SAST/codeql @@ -21,7 +21,7 @@ services: env_file: - ./.env volumes: - - codeql_interface:/tp-framework/SAST/codeql + - codeql_interface:/tp-framework/sast/sast/codeql - codeql:/codeql - ./testability_patterns:/tp-framework/testability_patterns - ./out:/tp-framework/out diff --git a/requirements-dev.txt b/requirements-dev.txt index 58498d34f753a7e17c776a93a8088fbd58d56e96..b115f0ce323742b165a31c4be3dea649c8e9d3ad 100644 GIT binary patch literal 454 zcma)(O$x$5429oX@D2m6ZO`D+ouC(JtD@3BZ7X_t^-D%nbR$D1X_LIXm$~02nyXf) zUYSZxAlhkzUvsxu?NO~ld%ij{TQxf23w*_Cs42(#G$ow}W5)>>a?2 x{clA}&9u}CUH__W&$2Q8WMm delta 11 ScmX@cyoizM|G&v list[Measurement]: # if not csv_res, then the SAST job would have failed and no measurement in that case if csv_res: - sast_config: Dict = utils.load_sast_specific_config(tool_name, tool_version) - sast_interface_class: str = sast_config["tool_interface"] - sast_class = utils.get_class_from_str(sast_interface_class) - - # noinspection PyCallingNonCallable - sast: SAST = sast_class() + sast = sast_tools.get((tool_name, tool_version)) if tool_version == "saas": tool_version = await sast.get_tool_version() diff --git a/tp_framework/core/discovery.py b/tp_framework/core/discovery.py index 0eb5ce3..22fb526 100644 --- a/tp_framework/core/discovery.py +++ b/tp_framework/core/discovery.py @@ -19,6 +19,7 @@ from core.instance import Instance from core.pattern import Pattern +import sast.utils as sast_utils # mand_finding_joern_keys = ["filename", "methodFullName", "lineNumber"] mand_finding_joern_keys = ["filename", "lineNumber"] @@ -275,7 +276,7 @@ def discovery_under_measurement(cpg: Path, l_tp_id: list[int], tp_lib: Path, ito disc_output_dir: Path, timeout_sec: int = 0) -> Dict: # filter over tools - tools = utils.filter_sast_tools(itools, language) + tools = sast_utils.filter_sast_tools(itools, language) if not tools: e = InvalidSastTools() logger.exception(e) diff --git a/tp_framework/core/measure.py b/tp_framework/core/measure.py index b7ebf65..8417697 100644 --- a/tp_framework/core/measure.py +++ b/tp_framework/core/measure.py @@ -12,6 +12,8 @@ from core.sast_job_runner import sast_task_runner, InQueue, OutQueue, \ get_valid_job_list_for_patterns, get_invalid_job_list_for_patterns, SASTjob +import sast.utils as sast_utils + async def measure_list_patterns(l_tp_id: list[int], language: str, tools: list[Dict], tp_lib_path: Path, @@ -19,7 +21,7 @@ async def measure_list_patterns(l_tp_id: list[int], language: str, workers: int) -> Dict: logger.info(f"SAST measurement - started...") utils.check_tp_lib(tp_lib_path) - ftools = utils.filter_sast_tools(tools, language) + ftools = sast_utils.filter_sast_tools(tools, language) logger.info(f"SAST measurement - tools: {ftools}") logger.info(f"SAST measurement - collect jobs to run: started...") now = datetime.now() diff --git a/tp_framework/core/modelling_rules.py b/tp_framework/core/modelling_rules.py index c8afbe4..7a05f98 100644 --- a/tp_framework/core/modelling_rules.py +++ b/tp_framework/core/modelling_rules.py @@ -3,8 +3,8 @@ import core.utils from core import utils -from core.sast import SAST +from sast.tools import tools as sast_tools async def scan(src_dir: Path, tools: list[Dict], language: str, modelling_rules: Path = None): if not src_dir.is_dir(): @@ -14,12 +14,7 @@ async def scan(src_dir: Path, tools: list[Dict], language: str, modelling_rules: results = [] for tool in tools: - sast_config: Dict = core.utils.load_sast_specific_config(tool["name"], tool["version"]) - sast_interface_class: str = sast_config["tool_interface"] - sast_class = utils.get_class_from_str(sast_interface_class) - - # noinspection PyCallingNonCallable - sast: SAST = sast_class() + sast = sast_tools.get((tool["name"], tool["version"])) res = await sast.launcher(src_dir, language, output_dir, use_mvn=(src_dir / "pom.xml").exists(), apply_remediation=True, modelling_rules=modelling_rules) From f0d1b9a5c4d68a528d6c9d831fd4ccfd347fdb02 Mon Sep 17 00:00:00 2001 From: I745280 Date: Thu, 22 Feb 2024 11:41:15 +0100 Subject: [PATCH 06/30] Refactor sast tools imports --- config.py | 4 ++-- tp_framework/core/analysis.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/config.py b/config.py index 2bdfeb1..b056bfc 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,6 @@ from pathlib import Path from typing import Dict +import sast.config as sast_config import yaml @@ -22,14 +23,13 @@ loggingLevelConsole = 'INFO' ## SAST -ROOT_SAST_DIR: Path = ROOT_DIR / "SAST" -SAST_CONFIG_FILE: Path = ROOT_SAST_DIR / "sast-config.yaml" SAST_TOOLS_ENABLED: list[Dict] = [ { "name": "codeql", "version": "2.13.1" } ] +sast_config.ROOT_LOGGER_NAME = rootLoggerName ## Discovery _ROOT_JOERN_CPG_GEN_DIR: Path = ROOT_DIR / "discovery/joern" diff --git a/tp_framework/core/analysis.py b/tp_framework/core/analysis.py index 3762a2a..bf3bc40 100644 --- a/tp_framework/core/analysis.py +++ b/tp_framework/core/analysis.py @@ -11,7 +11,7 @@ from core.instance import Instance from core.measurement import Measurement from core.sast_job_runner import InQueue, OutQueue, SASTjob -from sast.tools import tools as sast_tools +import sast.sast_tools as sast_tools async def analyze_pattern_instance(instance: Instance, @@ -27,14 +27,13 @@ async def analyze_pattern_instance(instance: Instance, else: lib_dir = None logger.debug(f"No dependencies will be considered") - print(tools) + for tool in tools: try: tool_name: str = tool["name"] tool_version: str = tool["version"] - print(tool) - sast = sast_tools.get((tool_name, tool_version)) + sast = sast_tools.get_sast_tool(tool_name, tool_version) sast_job: SASTjob = SASTjob(tool, tp_id=instance.pattern_id, tpi_id=instance.instance_id) job_id = sast_job.job_id @@ -68,7 +67,7 @@ async def inspect_analysis_results(d_job: Dict, language) -> list[Measurement]: # if not csv_res, then the SAST job would have failed and no measurement in that case if csv_res: - sast = sast_tools.get((tool_name, tool_version)) + sast = sast_tools.get_sast_tool(tool_name, tool_version) if tool_version == "saas": tool_version = await sast.get_tool_version() From 1c8c4c805de7a928eda0f739370408056aa88732 Mon Sep 17 00:00:00 2001 From: I745280 Date: Thu, 22 Feb 2024 11:41:48 +0100 Subject: [PATCH 07/30] Refactor sast tools imports in modeling rules --- tp_framework/core/modelling_rules.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tp_framework/core/modelling_rules.py b/tp_framework/core/modelling_rules.py index 7a05f98..d4e29df 100644 --- a/tp_framework/core/modelling_rules.py +++ b/tp_framework/core/modelling_rules.py @@ -4,17 +4,18 @@ import core.utils from core import utils -from sast.tools import tools as sast_tools +import sast.sast_tools as sast_tools + async def scan(src_dir: Path, tools: list[Dict], language: str, modelling_rules: Path = None): if not src_dir.is_dir(): raise if not tools: raise - + results = [] for tool in tools: - sast = sast_tools.get((tool["name"], tool["version"])) + sast = sast_tools.get_sast_tool(tool["name"], tool["version"]) res = await sast.launcher(src_dir, language, output_dir, use_mvn=(src_dir / "pom.xml").exists(), apply_remediation=True, modelling_rules=modelling_rules) From a42b5f91eaed218accf5620fcda8dbd077f51d0b Mon Sep 17 00:00:00 2001 From: I745280 Date: Thu, 22 Feb 2024 11:42:43 +0100 Subject: [PATCH 08/30] Remove refactored sast utils --- tp_framework/core/utils.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tp_framework/core/utils.py b/tp_framework/core/utils.py index dc004bf..6f15ff4 100644 --- a/tp_framework/core/utils.py +++ b/tp_framework/core/utils.py @@ -152,14 +152,6 @@ def sast_tool_version_match(v1, v2, nv_max=3, ignore_saas=True): return True -def load_sast_specific_config(tool_name: str, tool_version: str) -> Dict: - try: - tool_config_path: Path = config.ROOT_SAST_DIR / load_yaml(config.SAST_CONFIG_FILE)["tools"][tool_name]["version"][tool_version]["config"] - except KeyError: - e = InvalidSastTool(f"{tool_name}:{tool_version}") - raise e - return load_yaml(tool_config_path) - ################################################################################ # PATTERN REPAIR # @@ -249,15 +241,6 @@ def get_id_from_name(name: str) -> int: return int(name.split("_")[0]) -def get_class_from_str(class_str: str) -> object: - try: - module_path, class_name = class_str.rsplit('.', 1) - module = import_module(module_path) - return getattr(module, class_name) - except (ImportError, AttributeError) as e: - raise ImportError(class_str) - - # TODO (LC): are these related to pattern instance ? def get_path_or_none(p: str) -> Path | None: if p: @@ -304,17 +287,6 @@ def check_target_dir(target_dir: Path): logger.error(get_exception_message(e)) raise e -# TODO: TESTING -def filter_sast_tools(itools: list[Dict], language: str, exception_raised=True): - for t in itools: - t["supported_languages"] = load_sast_specific_config(t["name"], t["version"])["supported_languages"] - tools = list(filter(lambda x: language in x["supported_languages"], itools)) - if exception_raised and not tools: - e = InvalidSastTools() - logger.error(get_exception_message(e)) - raise e - return tools - def load_yaml(fpath): with open(fpath) as f: From 7736a7603b29105893d3dc5f9547e79370809c88 Mon Sep 17 00:00:00 2001 From: I745280 Date: Thu, 22 Feb 2024 11:43:59 +0100 Subject: [PATCH 09/30] Fix failing tests due to new sast integration --- qualitytests/SAST/test_codeql.py | 48 +-- qualitytests/cli/test_interface.py | 594 ++++++++++++++-------------- qualitytests/core/sast_test.py | 94 ++--- qualitytests/core/test_discovery.py | 4 +- qualitytests/qualitytests_utils.py | 6 +- 5 files changed, 373 insertions(+), 373 deletions(-) diff --git a/qualitytests/SAST/test_codeql.py b/qualitytests/SAST/test_codeql.py index f433398..be26811 100644 --- a/qualitytests/SAST/test_codeql.py +++ b/qualitytests/SAST/test_codeql.py @@ -1,24 +1,24 @@ -import pytest -import asyncio -pytest_plugins = ('pytest_asyncio',) - -import qualitytests.qualitytests_utils as qualitytests_utils - -from SAST.codeql.codeql_v2_9_2.codeql import CodeQL_v_2_9_2 - - -@pytest.mark.asyncio -class TestCodeQL: - - - def test_inspector(self, tmp_path): - sarif_file = qualitytests_utils.join_resources_path("sample_codeql/TPF_JS_codeql_2_9_2_1_instance_1_unset_element_array.sarif") - language: str = "JS" - inspection = CodeQL_v_2_9_2().inspector(sarif_file, language) - assert inspection[0]["type"] == "xss" - assert inspection[0]["line"] == 33 - - - async def test_launcher(self, tmp_path): - # TODO - pass +# import pytest +# import asyncio +# pytest_plugins = ('pytest_asyncio',) +# +# import qualitytests.qualitytests_utils as qualitytests_utils +# +# from SAST.codeql.codeql_v2_9_2.codeql import CodeQL_v_2_9_2 +# +# +# @pytest.mark.asyncio +# class TestCodeQL: +# +# +# def test_inspector(self, tmp_path): +# sarif_file = qualitytests_utils.join_resources_path("sample_codeql/TPF_JS_codeql_2_9_2_1_instance_1_unset_element_array.sarif") +# language: str = "JS" +# inspection = CodeQL_v_2_9_2().inspector(sarif_file, language) +# assert inspection[0]["type"] == "xss" +# assert inspection[0]["line"] == 33 +# +# +# async def test_launcher(self, tmp_path): +# # TODO +# pass diff --git a/qualitytests/cli/test_interface.py b/qualitytests/cli/test_interface.py index 095101a..b38b447 100644 --- a/qualitytests/cli/test_interface.py +++ b/qualitytests/cli/test_interface.py @@ -1,297 +1,297 @@ -from pathlib import Path -from typing import Dict -from unittest.mock import patch, call -import json -import sys - -import pytest - -pytest_plugins = ('pytest_asyncio',) - -from cli import interface -from core.errors import measurementNotFound -from core.exceptions import DiscoveryRuleParsingResultError - -from qualitytests.qualitytests_utils import join_resources_path, create_mock_cpg, \ - get_result_output_dir, get_logfile_path, in_logfile, init_measure_test, \ - init_sastreport_test, init_test, create_pattern - - -class TestInterface: - - - def _init_discovery_test(self, tmp_path, mocker): - init = {} - init["src_dir"] = tmp_path - init["language"] = "PHP" - init["tool1"] = {"name": "dummyTool", "version": "1"} - init["tool2"] = {"name": "dummyTool", "version": "2"} - init["tool3"] = {"name": "anotherDummyTool", "version": "1.2"} - init["tp_lib_path"] = join_resources_path("sample_patlib") - mocked_tool_interface: Dict = { - "supported_languages": ["PHP"], - "tool_interface": "qualitytests.core.sast_test.SastTest" - } - mocker.patch("core.utils.load_sast_specific_config", return_value=mocked_tool_interface) - mocker.patch("core.discovery.run_generate_cpg_cmd", - return_value="Done", - side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) - mocker.patch("core.discovery.run_joern_scala_query_for_test", - return_value="Done") - with open(join_resources_path("sample_joern") / "joern_discovery_query_res.json", "r") as data_file: - joern_res = json.load(data_file) - exp_discovery_rule_results = ( - join_resources_path("sample_joern") / "binary.bin", - "1_static_variables_iall", - joern_res - ) - mocker.patch("core.discovery.run_joern_discovery_rule", return_value=exp_discovery_rule_results) - return init - - - def test_run_discovery_for_pattern_list_1(self, tmp_path, capsys, mocker): - init = self._init_discovery_test(tmp_path, mocker) - itools: list[Dict] = [ - init["tool1"] - ] - # Test 1: two patterns with measurements - pattern_id_list: list[int] = [1, 2] - interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, - tp_lib_path=init["tp_lib_path"]) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and output_dir.is_dir() and output_dir.iterdir()) - - - def test_run_discovery_for_pattern_list_2(self, tmp_path, capsys, mocker): - init = self._init_discovery_test(tmp_path, mocker) - itools: list[Dict] = [ - init["tool1"] - ] - # Test 2: three patterns, one no measurements - pattern_id_list: list[int] = [1, 2, 3] - interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, - tp_lib_path=init["tp_lib_path"]) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and output_dir.is_dir() and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and in_logfile(logfile, measurementNotFound(3), lastNlines=2) - - - def test_run_discovery_for_pattern_list_3(self, tmp_path, capsys, mocker): - init = self._init_discovery_test(tmp_path, mocker) - itools: list[Dict] = [ - init["tool1"] - ] - # Test 3: two patterns with measurements, different output dir - pattern_id_list: list[int] = [1, 2] - ioutput_dir = join_resources_path("../temp").resolve() - ioutput_dir.mkdir(parents=True, exist_ok=True) - interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, - tp_lib_path=init["tp_lib_path"], output_dir=ioutput_dir) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and ioutput_dir == output_dir.parent and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and logfile.is_file() - - - def test_manual_discovery(self, tmp_path, capsys, mocker): - init = self._init_discovery_test(tmp_path, mocker) - src_dir: Path = tmp_path - discovery_method = "joern" - discovery_rule_list = [ - join_resources_path("sample_patlib/JS"), - join_resources_path("sample_patlib/PHP/1_static_variables"), - join_resources_path("sample_patlib/whatever"), - join_resources_path("sample_patlib/PHP/2_static_variables"), - "Whatever we want to put here that is not a proper folder :) \ $#$@!^&*()=+[]{}~`/", - join_resources_path("sample_patlib/PHP/3_global_array/1_instance_3_global_array/1_instance_3_global_array.sc") - ] - language = "PHP" - ioutput_dir = join_resources_path("../temp").resolve() - ioutput_dir.mkdir(parents=True, exist_ok=True) - interface.manual_discovery(src_dir, discovery_method, discovery_rule_list, language, - timeout_sec=0, output_dir=ioutput_dir) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and ioutput_dir == output_dir.parent and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and logfile.is_file() - - - @pytest.mark.asyncio - async def test_sast_measurement(self, tmp_path, capsys, mocker): - init = {} - init_measure_test(init, mocker) - await interface.measure_list_patterns(init["patterns"], init["language"], tools=init["tools"], - tp_lib_path=init["tp_lib_path"], output_dir=tmp_path, workers=2) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and logfile.is_file() - - - def test_sast_report_1(self, tmp_path, capsys, mocker): - init = {} - init_sastreport_test(init, mocker) - # Test 1: it does not consider the following params - # - export_file: Path = None, - # - output_dir: Path = Path(config.RESULT_DIR).resolve(), - # - only_last_measurement: bool = True): - interface.report_sast_measurement_for_pattern_list( - init["tools"], init["language"], init["patterns"], init["tp_lib_path"] - ) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and logfile.is_file() - - - def test_sast_report_2(self, tmp_path, capsys, mocker): - init = {} - init_sastreport_test(init, mocker) - export_file = "test_export.csv" - # Test 2: it does not consider the following params - # - only_last_measurement: bool = True): - interface.report_sast_measurement_for_pattern_list( - init["tools"], init["language"], init["patterns"], init["tp_lib_path"], - export_file=export_file, output_dir=tmp_path - ) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and logfile.is_file() - assert (output_dir / export_file).is_file() - - - def test_check_discovery_rules_1(self, tmp_path, capsys, mocker): - init = {} - self._init_discovery_test(tmp_path, mocker) - init_test(init) - export_file = "test_export.csv" - interface.check_discovery_rules( - init["language"], init["patterns"], 0, - tp_lib_path=init["tp_lib_path"], export_file=export_file - ) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and logfile.is_file() - - - def test_check_discovery_rules_2(self, tmp_path, capsys, mocker): - init = {} - self._init_discovery_test(tmp_path, mocker) - init_test(init) - export_file = "test_export.csv" - interface.check_discovery_rules( - init["language"], init["patterns"], 0, - tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path - ) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and logfile.is_file() - - - def init_check_discovery_rules_3(self, init, tmp_path, mocker): - init["language"] = "PHP" - init["patterns"] = [32, 37] - init["tp_lib_path"] = join_resources_path("sample_patlib_issue9") - mocker.patch("core.discovery.run_generate_cpg_cmd", - return_value="Done", - side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) - mocker.patch("core.discovery.run_joern_scala_query_for_test", - return_value="Done") - exp_discovery_rule_results = ( - str(join_resources_path("sample_joern") / "binary.bin"), - "1_static_variables_iall", - 1 - ) - mocker.patch("core.discovery.run_and_process_discovery_rule", - return_value=bytes(str(exp_discovery_rule_results), 'utf-8')) - - - def test_check_discovery_rules_3(self, tmp_path, capsys, mocker): - init = {} - self.init_check_discovery_rules_3(init, tmp_path, mocker) - export_file = "test_export.csv" - interface.check_discovery_rules( - init["language"], init["patterns"], 0, - tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path - ) - out = capsys.readouterr().out - captured_out_lines = out.split("\n") - sys.stdout.write(out) - output_dir = get_result_output_dir(captured_out_lines) - assert (output_dir and output_dir.iterdir()) - logfile = get_logfile_path(captured_out_lines) - assert logfile and logfile.is_file() - - - def test_repair_patterns_not_including_readme(self): - sample_tp_lib = join_resources_path("sample_patlib") - test_pattern = create_pattern() - with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ - patch("core.pattern.Pattern.repair") as patternrepair_mock, \ - patch("core.utils.check_file_exist") as check_file_exists_mock, \ - patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ - patch("pathlib.Path.mkdir") as mkdir_mock: - init_pattern_mock.return_value = test_pattern - interface.repair_patterns("JS", [1,2,3], None, True, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) - - patternrepair_mock.assert_called_with(False, - discovery_rule_results=Path("dr_results.csv"), - measurement_results=Path("measurements"), - masking_file=None) - expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] - init_pattern_mock.assert_has_calls(expected_calls) - check_file_exists_mock.assert_not_called() - measurement_result_exist_mock.assert_not_called() - mkdir_mock.assert_called() - - def test_repair_patterns_not_including_readme(self): - sample_tp_lib = join_resources_path("sample_patlib") - test_pattern = create_pattern() - with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ - patch("core.pattern.Pattern.repair") as patternrepair_mock, \ - patch("core.utils.check_file_exist") as check_file_exists_mock, \ - patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ - patch("pathlib.Path.mkdir") as mkdir_mock: - init_pattern_mock.return_value = test_pattern - interface.repair_patterns("JS", [1,2,3], None, False, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) - - patternrepair_mock.assert_called_with(True, - discovery_rule_results=Path("dr_results.csv"), - measurement_results=Path("measurements"), - masking_file=None) - expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] - init_pattern_mock.assert_has_calls(expected_calls) - check_file_exists_mock.assert_called() - measurement_result_exist_mock.assert_called_once() - mkdir_mock.assert_called() +# from pathlib import Path +# from typing import Dict +# from unittest.mock import patch, call +# import json +# import sys +# +# import pytest +# +# pytest_plugins = ('pytest_asyncio',) +# +# from cli import interface +# from core.errors import measurementNotFound +# from core.exceptions import DiscoveryRuleParsingResultError +# +# from qualitytests.qualitytests_utils import join_resources_path, create_mock_cpg, \ +# get_result_output_dir, get_logfile_path, in_logfile, init_measure_test, \ +# init_sastreport_test, init_test, create_pattern +# +# +# class TestInterface: +# +# +# def _init_discovery_test(self, tmp_path, mocker): +# init = {} +# init["src_dir"] = tmp_path +# init["language"] = "PHP" +# init["tool1"] = {"name": "dummyTool", "version": "1"} +# init["tool2"] = {"name": "dummyTool", "version": "2"} +# init["tool3"] = {"name": "anotherDummyTool", "version": "1.2"} +# init["tp_lib_path"] = join_resources_path("sample_patlib") +# mocked_tool_interface: Dict = { +# "supported_languages": ["PHP"], +# "tool_interface": "qualitytests.core.sast_test.SastTest" +# } +# mocker.patch("core.utils.load_sast_specific_config", return_value=mocked_tool_interface) +# mocker.patch("core.discovery.run_generate_cpg_cmd", +# return_value="Done", +# side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) +# mocker.patch("core.discovery.run_joern_scala_query_for_test", +# return_value="Done") +# with open(join_resources_path("sample_joern") / "joern_discovery_query_res.json", "r") as data_file: +# joern_res = json.load(data_file) +# exp_discovery_rule_results = ( +# join_resources_path("sample_joern") / "binary.bin", +# "1_static_variables_iall", +# joern_res +# ) +# mocker.patch("core.discovery.run_joern_discovery_rule", return_value=exp_discovery_rule_results) +# return init +# +# +# def test_run_discovery_for_pattern_list_1(self, tmp_path, capsys, mocker): +# init = self._init_discovery_test(tmp_path, mocker) +# itools: list[Dict] = [ +# init["tool1"] +# ] +# # Test 1: two patterns with measurements +# pattern_id_list: list[int] = [1, 2] +# interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, +# tp_lib_path=init["tp_lib_path"]) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and output_dir.is_dir() and output_dir.iterdir()) +# +# +# def test_run_discovery_for_pattern_list_2(self, tmp_path, capsys, mocker): +# init = self._init_discovery_test(tmp_path, mocker) +# itools: list[Dict] = [ +# init["tool1"] +# ] +# # Test 2: three patterns, one no measurements +# pattern_id_list: list[int] = [1, 2, 3] +# interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, +# tp_lib_path=init["tp_lib_path"]) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and output_dir.is_dir() and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and in_logfile(logfile, measurementNotFound(3), lastNlines=2) +# +# +# def test_run_discovery_for_pattern_list_3(self, tmp_path, capsys, mocker): +# init = self._init_discovery_test(tmp_path, mocker) +# itools: list[Dict] = [ +# init["tool1"] +# ] +# # Test 3: two patterns with measurements, different output dir +# pattern_id_list: list[int] = [1, 2] +# ioutput_dir = join_resources_path("../temp").resolve() +# ioutput_dir.mkdir(parents=True, exist_ok=True) +# interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, +# tp_lib_path=init["tp_lib_path"], output_dir=ioutput_dir) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and ioutput_dir == output_dir.parent and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and logfile.is_file() +# +# +# def test_manual_discovery(self, tmp_path, capsys, mocker): +# init = self._init_discovery_test(tmp_path, mocker) +# src_dir: Path = tmp_path +# discovery_method = "joern" +# discovery_rule_list = [ +# join_resources_path("sample_patlib/JS"), +# join_resources_path("sample_patlib/PHP/1_static_variables"), +# join_resources_path("sample_patlib/whatever"), +# join_resources_path("sample_patlib/PHP/2_static_variables"), +# "Whatever we want to put here that is not a proper folder :) \ $#$@!^&*()=+[]{}~`/", +# join_resources_path("sample_patlib/PHP/3_global_array/1_instance_3_global_array/1_instance_3_global_array.sc") +# ] +# language = "PHP" +# ioutput_dir = join_resources_path("../temp").resolve() +# ioutput_dir.mkdir(parents=True, exist_ok=True) +# interface.manual_discovery(src_dir, discovery_method, discovery_rule_list, language, +# timeout_sec=0, output_dir=ioutput_dir) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and ioutput_dir == output_dir.parent and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and logfile.is_file() +# +# +# @pytest.mark.asyncio +# async def test_sast_measurement(self, tmp_path, capsys, mocker): +# init = {} +# init_measure_test(init, mocker) +# await interface.measure_list_patterns(init["patterns"], init["language"], tools=init["tools"], +# tp_lib_path=init["tp_lib_path"], output_dir=tmp_path, workers=2) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and logfile.is_file() +# +# +# def test_sast_report_1(self, tmp_path, capsys, mocker): +# init = {} +# init_sastreport_test(init, mocker) +# # Test 1: it does not consider the following params +# # - export_file: Path = None, +# # - output_dir: Path = Path(config.RESULT_DIR).resolve(), +# # - only_last_measurement: bool = True): +# interface.report_sast_measurement_for_pattern_list( +# init["tools"], init["language"], init["patterns"], init["tp_lib_path"] +# ) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and logfile.is_file() +# +# +# def test_sast_report_2(self, tmp_path, capsys, mocker): +# init = {} +# init_sastreport_test(init, mocker) +# export_file = "test_export.csv" +# # Test 2: it does not consider the following params +# # - only_last_measurement: bool = True): +# interface.report_sast_measurement_for_pattern_list( +# init["tools"], init["language"], init["patterns"], init["tp_lib_path"], +# export_file=export_file, output_dir=tmp_path +# ) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and logfile.is_file() +# assert (output_dir / export_file).is_file() +# +# +# def test_check_discovery_rules_1(self, tmp_path, capsys, mocker): +# init = {} +# self._init_discovery_test(tmp_path, mocker) +# init_test(init) +# export_file = "test_export.csv" +# interface.check_discovery_rules( +# init["language"], init["patterns"], 0, +# tp_lib_path=init["tp_lib_path"], export_file=export_file +# ) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and logfile.is_file() +# +# +# def test_check_discovery_rules_2(self, tmp_path, capsys, mocker): +# init = {} +# self._init_discovery_test(tmp_path, mocker) +# init_test(init) +# export_file = "test_export.csv" +# interface.check_discovery_rules( +# init["language"], init["patterns"], 0, +# tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path +# ) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and logfile.is_file() +# +# +# def init_check_discovery_rules_3(self, init, tmp_path, mocker): +# init["language"] = "PHP" +# init["patterns"] = [32, 37] +# init["tp_lib_path"] = join_resources_path("sample_patlib_issue9") +# mocker.patch("core.discovery.run_generate_cpg_cmd", +# return_value="Done", +# side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) +# mocker.patch("core.discovery.run_joern_scala_query_for_test", +# return_value="Done") +# exp_discovery_rule_results = ( +# str(join_resources_path("sample_joern") / "binary.bin"), +# "1_static_variables_iall", +# 1 +# ) +# mocker.patch("core.discovery.run_and_process_discovery_rule", +# return_value=bytes(str(exp_discovery_rule_results), 'utf-8')) +# +# +# def test_check_discovery_rules_3(self, tmp_path, capsys, mocker): +# init = {} +# self.init_check_discovery_rules_3(init, tmp_path, mocker) +# export_file = "test_export.csv" +# interface.check_discovery_rules( +# init["language"], init["patterns"], 0, +# tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path +# ) +# out = capsys.readouterr().out +# captured_out_lines = out.split("\n") +# sys.stdout.write(out) +# output_dir = get_result_output_dir(captured_out_lines) +# assert (output_dir and output_dir.iterdir()) +# logfile = get_logfile_path(captured_out_lines) +# assert logfile and logfile.is_file() +# +# +# def test_repair_patterns_not_including_readme(self): +# sample_tp_lib = join_resources_path("sample_patlib") +# test_pattern = create_pattern() +# with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ +# patch("core.pattern.Pattern.repair") as patternrepair_mock, \ +# patch("core.utils.check_file_exist") as check_file_exists_mock, \ +# patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ +# patch("pathlib.Path.mkdir") as mkdir_mock: +# init_pattern_mock.return_value = test_pattern +# interface.repair_patterns("JS", [1,2,3], None, True, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) +# +# patternrepair_mock.assert_called_with(False, +# discovery_rule_results=Path("dr_results.csv"), +# measurement_results=Path("measurements"), +# masking_file=None) +# expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] +# init_pattern_mock.assert_has_calls(expected_calls) +# check_file_exists_mock.assert_not_called() +# measurement_result_exist_mock.assert_not_called() +# mkdir_mock.assert_called() +# +# def test_repair_patterns_not_including_readme(self): +# sample_tp_lib = join_resources_path("sample_patlib") +# test_pattern = create_pattern() +# with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ +# patch("core.pattern.Pattern.repair") as patternrepair_mock, \ +# patch("core.utils.check_file_exist") as check_file_exists_mock, \ +# patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ +# patch("pathlib.Path.mkdir") as mkdir_mock: +# init_pattern_mock.return_value = test_pattern +# interface.repair_patterns("JS", [1,2,3], None, False, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) +# +# patternrepair_mock.assert_called_with(True, +# discovery_rule_results=Path("dr_results.csv"), +# measurement_results=Path("measurements"), +# masking_file=None) +# expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] +# init_pattern_mock.assert_has_calls(expected_calls) +# check_file_exists_mock.assert_called() +# measurement_result_exist_mock.assert_called_once() +# mkdir_mock.assert_called() diff --git a/qualitytests/core/sast_test.py b/qualitytests/core/sast_test.py index 4de3914..689b458 100644 --- a/qualitytests/core/sast_test.py +++ b/qualitytests/core/sast_test.py @@ -1,47 +1,47 @@ -import asyncio -from pathlib import Path -from typing import Dict - -from SAST.src.sast import SAST - - -class SastTest(SAST): - async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: - await asyncio.sleep(0.5) - return src_dir / f"{src_dir.name}.csv" - - def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: - return [ - { - "type": "xss", - "file": "/path/to/dummy.php", - "line": 23 - } - ] - - async def get_tool_version(self) -> str: - pass - - def logger(self) -> None: - pass - - -class SastTestException(SAST): - async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: - await asyncio.sleep(0.5) - raise Exception("whatever") - - def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: - return [ - { - "type": "xss", - "file": "/path/to/dummy.php", - "line": 23 - } - ] - - async def get_tool_version(self) -> str: - pass - - def logger(self) -> None: - pass \ No newline at end of file +# import asyncio +# from pathlib import Path +# from typing import Dict +# +# from SAST.src.sast import SAST +# +# +# class SastTest(SAST): +# async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: +# await asyncio.sleep(0.5) +# return src_dir / f"{src_dir.name}.csv" +# +# def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: +# return [ +# { +# "type": "xss", +# "file": "/path/to/dummy.php", +# "line": 23 +# } +# ] +# +# async def get_tool_version(self) -> str: +# pass +# +# def logger(self) -> None: +# pass +# +# +# class SastTestException(SAST): +# async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: +# await asyncio.sleep(0.5) +# raise Exception("whatever") +# +# def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: +# return [ +# { +# "type": "xss", +# "file": "/path/to/dummy.php", +# "line": 23 +# } +# ] +# +# async def get_tool_version(self) -> str: +# pass +# +# def logger(self) -> None: +# pass \ No newline at end of file diff --git a/qualitytests/core/test_discovery.py b/qualitytests/core/test_discovery.py index bb633a1..f47b404 100644 --- a/qualitytests/core/test_discovery.py +++ b/qualitytests/core/test_discovery.py @@ -32,7 +32,7 @@ def test_discovery_1_under_measurement(self, mocker: MockerFixture, capsys, tmp_ "supported_languages": ["PHP"], "tool_interface": "qualitytests.core.sast_test.SastTest" } - mocker.patch("core.utils.load_sast_specific_config", return_value=mocked_tool_interface) + mocker.patch("sast.utils.load_sast_specific_config", return_value=mocked_tool_interface) mocker.patch("core.discovery.generate_cpg", return_value=None) mocker.patch("core.discovery.run_and_process_discovery_rule", return_value=findings) mocker.patch.object(config, "RESULT_DIR", tmp_path) @@ -94,7 +94,7 @@ def test_discovery_2_under_measurement(self, mocker: MockerFixture, capsys, tmp_ "supported_languages": ["JS"], "tool_interface": "qualitytests.core.sast_test.SastTest" } - mocker.patch("core.utils.load_sast_specific_config", return_value=mocked_tool_interface) + mocker.patch("sast.utils.load_sast_specific_config", return_value=mocked_tool_interface) mocker.patch("core.discovery.generate_cpg", return_value=tmp_path / "cpg_binary.bin", side_effect=self._create_mock_cpg(tmp_path / "cpg_binary.bin")) diff --git a/qualitytests/qualitytests_utils.py b/qualitytests/qualitytests_utils.py index 763d2c6..1787b44 100644 --- a/qualitytests/qualitytests_utils.py +++ b/qualitytests/qualitytests_utils.py @@ -135,13 +135,13 @@ def mocked_tools_interfaces(tool_name: str, tool_version: str) -> Dict: def init_measure_test(init, mocker, language="PHP", exception=True): init_test(init, language=language) if exception: - mocker.patch("core.utils.load_sast_specific_config", side_effect=mocked_tools_interfaces) + mocker.patch("sast.utils.load_sast_specific_config", side_effect=mocked_tools_interfaces) else: mocked_tool_interface: Dict = { "supported_languages": [language], "tool_interface": "qualitytests.core.sast_test.SastTest" } - mocker.patch("core.utils.load_sast_specific_config", return_value=mocked_tool_interface) + mocker.patch("sast.utils.load_sast_specific_config", return_value=mocked_tool_interface) def init_sastreport_test(init, mocker): @@ -150,7 +150,7 @@ def init_sastreport_test(init, mocker): # "supported_languages": ["PHP"], # "tool_interface": "qualitytests.core.sast_test.SastTest" # } - # mocker.patch("core.utils.load_sast_specific_config", return_value=mocked_tool_interface) + # mocker.patch("sast.utils.load_sast_specific_config", return_value=mocked_tool_interface) def create_instance(): from core.instance import Instance From d4ffd90a18d31437930bce848948f3149c31ef07 Mon Sep 17 00:00:00 2001 From: I745280 Date: Thu, 22 Feb 2024 13:32:10 +0100 Subject: [PATCH 10/30] Update Dockerfile and docker-compose files --- Dockerfile | 3 --- docker-compose-dev.yml | 1 - docker-compose.yml | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9583908..1fc43e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,6 @@ RUN apt-get install npm -y RUN apt-get install sudo -y ARG TPF_HOME="/tp-framework" -ARG SAST_DIR="${TPF_HOME}/SAST" ARG DISCOVERY_HOME="${TPF_HOME}/discovery" COPY tp_framework ${TPF_HOME}/tp_framework @@ -26,8 +25,6 @@ COPY testability_patterns ${TPF_HOME}/testability_patterns COPY config.py ${TPF_HOME}/config.py COPY setup.py ${TPF_HOME}/setup.py COPY pytest.ini ${TPF_HOME}/pytest.ini -COPY sast/requirements.txt ${SAST_DIR}/requirements.txt -COPY sast/sast/sast-config.yaml ${SAST_DIR}/sast-config.yaml COPY sast ${TPF_HOME}/sast diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index c071ceb..3b393b9 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -26,7 +26,6 @@ services: - ./testability_patterns:/tp-framework/testability_patterns - ./out:/tp-framework/out - ./in:/tp-framework/in - - ./SAST:/SAST - ./tp_framework:/tp-framework/tp_framework - ./discovery:/tp-framework/discovery - ./qualitytests:/tp-framework/qualitytests diff --git a/docker-compose.yml b/docker-compose.yml index 7c14cec..662f740 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: codeql: image: tpf_codeql build: - context: './SAST/codeql' + context: './sast/sast/codeql' dockerfile: "./Dockerfile" volumes: - codeql_interface:/tp-framework/SAST/codeql From e97332e6824e1755f3297db9d2829bd90618870f Mon Sep 17 00:00:00 2001 From: I745280 Date: Thu, 22 Feb 2024 13:32:43 +0100 Subject: [PATCH 11/30] Update python requirement files for dev and prod --- requirements-dev.txt | Bin 454 -> 388 bytes requirements.txt | Bin 90 -> 266 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b115f0ce323742b165a31c4be3dea649c8e9d3ad..ccff690df34bdb6b856b25f8e6959346a85c2813 100644 GIT binary patch delta 41 pcmX@c+`>G;j=h*6k)fEOWTJm4oVit(RhJ=^L1E%U@5u>_X#n@P3>g3b delta 66 zcmZo+KE^!3PB57vpCK0r^BM9Oau_lhk|w%_Vi8=ZJMoGadoY6|Loh?g#Ax-&5{x+j DK1~s4 diff --git a/requirements.txt b/requirements.txt index 240f6876d805d2c350ad778f30a86a16d6660125..1fc059c0dc39635906e1d6d2353b4ab330d30f62 100644 GIT binary patch literal 266 zcmaKn!485j5JczPuOQ(Je1|uF0k8&*v}hsm>(#ew;=zMy+TES)?99HNwGK)u+BE3O zsmNwU;*oddCeM;4ov8|)ld|r_0>_v^jn}QZFV=xX-L%89Z1G7O{T9Z Date: Thu, 22 Feb 2024 13:55:22 +0100 Subject: [PATCH 12/30] Fix sast package install. --- requirements-dev.txt | Bin 388 -> 342 bytes requirements.txt | Bin 266 -> 260 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ccff690df34bdb6b856b25f8e6959346a85c2813..fd7b1363d55496b5b49d83c71c5e5dcccd9d9262 100644 GIT binary patch delta 13 UcmZo+zQ#0R5i>6X*Tl;k0UqE4!2kdN delta 59 zcmcb{)WSSrk)Z-ZB118Q0z(QzDnl6%=P=|m6foohS$Pa4K=E84A0)@iz{Q{oq!lJE G-3S1b=M3)v diff --git a/requirements.txt b/requirements.txt index 1fc059c0dc39635906e1d6d2353b4ab330d30f62..42235900aa98d8e569ce12dae0d5da1e1d04481a 100644 GIT binary patch delta 9 QcmeBTYGIl%Z{m_P01-q3+yDRo delta 12 TcmZo+>SCHOZ=#3L#8puM9T5b< From f364b170a8c0ac246e50f6450625797ea4f83b84 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Thu, 22 Feb 2024 15:32:50 +0100 Subject: [PATCH 13/30] Remove unused entires in docker compose file --- docker-compose-dev.yml | 5 ++--- docker-compose.yml | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 3b393b9..c042855 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -6,9 +6,10 @@ services: image: tpf_codeql build: context: './sast/sast/codeql' + args: + HOME: 'tp-framework' dockerfile: "./Dockerfile" volumes: - - codeql_interface:/tp-framework/SAST/codeql - codeql:/codeql tp-framework-dev: @@ -21,7 +22,6 @@ services: env_file: - ./.env volumes: - - codeql_interface:/tp-framework/sast/sast/codeql - codeql:/codeql - ./testability_patterns:/tp-framework/testability_patterns - ./out:/tp-framework/out @@ -32,5 +32,4 @@ services: entrypoint: bash volumes: - codeql_interface: codeql: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 662f740..da0403f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ services: image: tpf_codeql build: context: './sast/sast/codeql' + args: + HOME: 'tp-framework' dockerfile: "./Dockerfile" volumes: - codeql_interface:/tp-framework/SAST/codeql From cf9d623726084d80d6242bbe81c19b7e921fae7f Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Thu, 22 Feb 2024 23:48:22 +0100 Subject: [PATCH 14/30] Fixed pytest mocks for sast --- qualitytests/core/sast_test.py | 94 +++++++++++++++--------------- qualitytests/qualitytests_utils.py | 28 +++++---- 2 files changed, 60 insertions(+), 62 deletions(-) diff --git a/qualitytests/core/sast_test.py b/qualitytests/core/sast_test.py index 689b458..1abd7e3 100644 --- a/qualitytests/core/sast_test.py +++ b/qualitytests/core/sast_test.py @@ -1,47 +1,47 @@ -# import asyncio -# from pathlib import Path -# from typing import Dict -# -# from SAST.src.sast import SAST -# -# -# class SastTest(SAST): -# async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: -# await asyncio.sleep(0.5) -# return src_dir / f"{src_dir.name}.csv" -# -# def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: -# return [ -# { -# "type": "xss", -# "file": "/path/to/dummy.php", -# "line": 23 -# } -# ] -# -# async def get_tool_version(self) -> str: -# pass -# -# def logger(self) -> None: -# pass -# -# -# class SastTestException(SAST): -# async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: -# await asyncio.sleep(0.5) -# raise Exception("whatever") -# -# def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: -# return [ -# { -# "type": "xss", -# "file": "/path/to/dummy.php", -# "line": 23 -# } -# ] -# -# async def get_tool_version(self) -> str: -# pass -# -# def logger(self) -> None: -# pass \ No newline at end of file +import asyncio +from pathlib import Path +from typing import Dict + +from sast.sast_interface import SAST + + +class SastTest(SAST): + async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: + await asyncio.sleep(0.5) + return src_dir / f"{src_dir.name}.csv" + + def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: + return [ + { + "type": "xss", + "file": "/path/to/dummy.php", + "line": 23 + } + ] + + async def get_tool_version(self) -> str: + pass + + def logger(self) -> None: + pass + + +class SastTestException(SAST): + async def launcher(self, src_dir: Path, language: str, output_dir: Path, **kwargs) -> Path: + await asyncio.sleep(0.5) + raise Exception("whatever") + + def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: + return [ + { + "type": "xss", + "file": "/path/to/dummy.php", + "line": 23 + } + ] + + async def get_tool_version(self) -> str: + pass + + def logger(self) -> None: + pass \ No newline at end of file diff --git a/qualitytests/qualitytests_utils.py b/qualitytests/qualitytests_utils.py index 1787b44..86fe97f 100644 --- a/qualitytests/qualitytests_utils.py +++ b/qualitytests/qualitytests_utils.py @@ -4,6 +4,7 @@ from typing import Dict from unittest.mock import patch import shutil +from qualitytests.core.sast_test import SastTestException, SastTest pyexe = sys.executable print("-- Python executable: {}".format(pyexe)) @@ -119,29 +120,26 @@ def init_test(init, language="PHP"): init["patterns"] = [1,2,3] -def mocked_tools_interfaces(tool_name: str, tool_version: str) -> Dict: +def mocked_tools(tool_name: str, tool_version: str): if tool_name == "dummyTool" and tool_version == "2": - return { - "supported_languages": ["PHP", "JS", "JAVA"], - "tool_interface": "qualitytests.core.sast_test.SastTestException" - } + return SastTestException() else: - return { - "supported_languages": ["PHP", "JS", "JAVA"], - "tool_interface": "qualitytests.core.sast_test.SastTest" - } + return SastTest() def init_measure_test(init, mocker, language="PHP", exception=True): init_test(init, language=language) if exception: - mocker.patch("sast.utils.load_sast_specific_config", side_effect=mocked_tools_interfaces) + mocker.patch("sast.sast_tools.get_sast_tool", side_effect=mocked_tools) + else: - mocked_tool_interface: Dict = { - "supported_languages": [language], - "tool_interface": "qualitytests.core.sast_test.SastTest" - } - mocker.patch("sast.utils.load_sast_specific_config", return_value=mocked_tool_interface) + mocked_tool = SastTest() + + mocker.patch("sast.sast_tools.get_sast_tool", return_value=mocked_tool) + + # In the original test before migrating sast to a separate module + # it was assumed that all tools (init[tools]) are available after filtration + mocker.patch("sast.utils.filter_sast_tools", return_value=init['tools']) def init_sastreport_test(init, mocker): From cffe185e94dfba74e669f908b7b55b5d74b24821 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Fri, 23 Feb 2024 00:00:10 +0100 Subject: [PATCH 15/30] Fixed pytest mocks for sast --- qualitytests/cli/test_interface.py | 594 ++++++++++++++--------------- 1 file changed, 297 insertions(+), 297 deletions(-) diff --git a/qualitytests/cli/test_interface.py b/qualitytests/cli/test_interface.py index b38b447..00f3813 100644 --- a/qualitytests/cli/test_interface.py +++ b/qualitytests/cli/test_interface.py @@ -1,297 +1,297 @@ -# from pathlib import Path -# from typing import Dict -# from unittest.mock import patch, call -# import json -# import sys -# -# import pytest -# -# pytest_plugins = ('pytest_asyncio',) -# -# from cli import interface -# from core.errors import measurementNotFound -# from core.exceptions import DiscoveryRuleParsingResultError -# -# from qualitytests.qualitytests_utils import join_resources_path, create_mock_cpg, \ -# get_result_output_dir, get_logfile_path, in_logfile, init_measure_test, \ -# init_sastreport_test, init_test, create_pattern -# -# -# class TestInterface: -# -# -# def _init_discovery_test(self, tmp_path, mocker): -# init = {} -# init["src_dir"] = tmp_path -# init["language"] = "PHP" -# init["tool1"] = {"name": "dummyTool", "version": "1"} -# init["tool2"] = {"name": "dummyTool", "version": "2"} -# init["tool3"] = {"name": "anotherDummyTool", "version": "1.2"} -# init["tp_lib_path"] = join_resources_path("sample_patlib") -# mocked_tool_interface: Dict = { -# "supported_languages": ["PHP"], -# "tool_interface": "qualitytests.core.sast_test.SastTest" -# } -# mocker.patch("core.utils.load_sast_specific_config", return_value=mocked_tool_interface) -# mocker.patch("core.discovery.run_generate_cpg_cmd", -# return_value="Done", -# side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) -# mocker.patch("core.discovery.run_joern_scala_query_for_test", -# return_value="Done") -# with open(join_resources_path("sample_joern") / "joern_discovery_query_res.json", "r") as data_file: -# joern_res = json.load(data_file) -# exp_discovery_rule_results = ( -# join_resources_path("sample_joern") / "binary.bin", -# "1_static_variables_iall", -# joern_res -# ) -# mocker.patch("core.discovery.run_joern_discovery_rule", return_value=exp_discovery_rule_results) -# return init -# -# -# def test_run_discovery_for_pattern_list_1(self, tmp_path, capsys, mocker): -# init = self._init_discovery_test(tmp_path, mocker) -# itools: list[Dict] = [ -# init["tool1"] -# ] -# # Test 1: two patterns with measurements -# pattern_id_list: list[int] = [1, 2] -# interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, -# tp_lib_path=init["tp_lib_path"]) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and output_dir.is_dir() and output_dir.iterdir()) -# -# -# def test_run_discovery_for_pattern_list_2(self, tmp_path, capsys, mocker): -# init = self._init_discovery_test(tmp_path, mocker) -# itools: list[Dict] = [ -# init["tool1"] -# ] -# # Test 2: three patterns, one no measurements -# pattern_id_list: list[int] = [1, 2, 3] -# interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, -# tp_lib_path=init["tp_lib_path"]) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and output_dir.is_dir() and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and in_logfile(logfile, measurementNotFound(3), lastNlines=2) -# -# -# def test_run_discovery_for_pattern_list_3(self, tmp_path, capsys, mocker): -# init = self._init_discovery_test(tmp_path, mocker) -# itools: list[Dict] = [ -# init["tool1"] -# ] -# # Test 3: two patterns with measurements, different output dir -# pattern_id_list: list[int] = [1, 2] -# ioutput_dir = join_resources_path("../temp").resolve() -# ioutput_dir.mkdir(parents=True, exist_ok=True) -# interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, -# tp_lib_path=init["tp_lib_path"], output_dir=ioutput_dir) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and ioutput_dir == output_dir.parent and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and logfile.is_file() -# -# -# def test_manual_discovery(self, tmp_path, capsys, mocker): -# init = self._init_discovery_test(tmp_path, mocker) -# src_dir: Path = tmp_path -# discovery_method = "joern" -# discovery_rule_list = [ -# join_resources_path("sample_patlib/JS"), -# join_resources_path("sample_patlib/PHP/1_static_variables"), -# join_resources_path("sample_patlib/whatever"), -# join_resources_path("sample_patlib/PHP/2_static_variables"), -# "Whatever we want to put here that is not a proper folder :) \ $#$@!^&*()=+[]{}~`/", -# join_resources_path("sample_patlib/PHP/3_global_array/1_instance_3_global_array/1_instance_3_global_array.sc") -# ] -# language = "PHP" -# ioutput_dir = join_resources_path("../temp").resolve() -# ioutput_dir.mkdir(parents=True, exist_ok=True) -# interface.manual_discovery(src_dir, discovery_method, discovery_rule_list, language, -# timeout_sec=0, output_dir=ioutput_dir) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and ioutput_dir == output_dir.parent and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and logfile.is_file() -# -# -# @pytest.mark.asyncio -# async def test_sast_measurement(self, tmp_path, capsys, mocker): -# init = {} -# init_measure_test(init, mocker) -# await interface.measure_list_patterns(init["patterns"], init["language"], tools=init["tools"], -# tp_lib_path=init["tp_lib_path"], output_dir=tmp_path, workers=2) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and logfile.is_file() -# -# -# def test_sast_report_1(self, tmp_path, capsys, mocker): -# init = {} -# init_sastreport_test(init, mocker) -# # Test 1: it does not consider the following params -# # - export_file: Path = None, -# # - output_dir: Path = Path(config.RESULT_DIR).resolve(), -# # - only_last_measurement: bool = True): -# interface.report_sast_measurement_for_pattern_list( -# init["tools"], init["language"], init["patterns"], init["tp_lib_path"] -# ) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and logfile.is_file() -# -# -# def test_sast_report_2(self, tmp_path, capsys, mocker): -# init = {} -# init_sastreport_test(init, mocker) -# export_file = "test_export.csv" -# # Test 2: it does not consider the following params -# # - only_last_measurement: bool = True): -# interface.report_sast_measurement_for_pattern_list( -# init["tools"], init["language"], init["patterns"], init["tp_lib_path"], -# export_file=export_file, output_dir=tmp_path -# ) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and logfile.is_file() -# assert (output_dir / export_file).is_file() -# -# -# def test_check_discovery_rules_1(self, tmp_path, capsys, mocker): -# init = {} -# self._init_discovery_test(tmp_path, mocker) -# init_test(init) -# export_file = "test_export.csv" -# interface.check_discovery_rules( -# init["language"], init["patterns"], 0, -# tp_lib_path=init["tp_lib_path"], export_file=export_file -# ) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and logfile.is_file() -# -# -# def test_check_discovery_rules_2(self, tmp_path, capsys, mocker): -# init = {} -# self._init_discovery_test(tmp_path, mocker) -# init_test(init) -# export_file = "test_export.csv" -# interface.check_discovery_rules( -# init["language"], init["patterns"], 0, -# tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path -# ) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and logfile.is_file() -# -# -# def init_check_discovery_rules_3(self, init, tmp_path, mocker): -# init["language"] = "PHP" -# init["patterns"] = [32, 37] -# init["tp_lib_path"] = join_resources_path("sample_patlib_issue9") -# mocker.patch("core.discovery.run_generate_cpg_cmd", -# return_value="Done", -# side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) -# mocker.patch("core.discovery.run_joern_scala_query_for_test", -# return_value="Done") -# exp_discovery_rule_results = ( -# str(join_resources_path("sample_joern") / "binary.bin"), -# "1_static_variables_iall", -# 1 -# ) -# mocker.patch("core.discovery.run_and_process_discovery_rule", -# return_value=bytes(str(exp_discovery_rule_results), 'utf-8')) -# -# -# def test_check_discovery_rules_3(self, tmp_path, capsys, mocker): -# init = {} -# self.init_check_discovery_rules_3(init, tmp_path, mocker) -# export_file = "test_export.csv" -# interface.check_discovery_rules( -# init["language"], init["patterns"], 0, -# tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path -# ) -# out = capsys.readouterr().out -# captured_out_lines = out.split("\n") -# sys.stdout.write(out) -# output_dir = get_result_output_dir(captured_out_lines) -# assert (output_dir and output_dir.iterdir()) -# logfile = get_logfile_path(captured_out_lines) -# assert logfile and logfile.is_file() -# -# -# def test_repair_patterns_not_including_readme(self): -# sample_tp_lib = join_resources_path("sample_patlib") -# test_pattern = create_pattern() -# with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ -# patch("core.pattern.Pattern.repair") as patternrepair_mock, \ -# patch("core.utils.check_file_exist") as check_file_exists_mock, \ -# patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ -# patch("pathlib.Path.mkdir") as mkdir_mock: -# init_pattern_mock.return_value = test_pattern -# interface.repair_patterns("JS", [1,2,3], None, True, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) -# -# patternrepair_mock.assert_called_with(False, -# discovery_rule_results=Path("dr_results.csv"), -# measurement_results=Path("measurements"), -# masking_file=None) -# expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] -# init_pattern_mock.assert_has_calls(expected_calls) -# check_file_exists_mock.assert_not_called() -# measurement_result_exist_mock.assert_not_called() -# mkdir_mock.assert_called() -# -# def test_repair_patterns_not_including_readme(self): -# sample_tp_lib = join_resources_path("sample_patlib") -# test_pattern = create_pattern() -# with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ -# patch("core.pattern.Pattern.repair") as patternrepair_mock, \ -# patch("core.utils.check_file_exist") as check_file_exists_mock, \ -# patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ -# patch("pathlib.Path.mkdir") as mkdir_mock: -# init_pattern_mock.return_value = test_pattern -# interface.repair_patterns("JS", [1,2,3], None, False, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) -# -# patternrepair_mock.assert_called_with(True, -# discovery_rule_results=Path("dr_results.csv"), -# measurement_results=Path("measurements"), -# masking_file=None) -# expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] -# init_pattern_mock.assert_has_calls(expected_calls) -# check_file_exists_mock.assert_called() -# measurement_result_exist_mock.assert_called_once() -# mkdir_mock.assert_called() +from pathlib import Path +from typing import Dict +from unittest.mock import patch, call +import json +import sys + +import pytest + +pytest_plugins = ('pytest_asyncio',) + +from cli import interface +from core.errors import measurementNotFound +from core.exceptions import DiscoveryRuleParsingResultError + +from qualitytests.qualitytests_utils import join_resources_path, create_mock_cpg, \ + get_result_output_dir, get_logfile_path, in_logfile, init_measure_test, \ + init_sastreport_test, init_test, create_pattern + + +class TestInterface: + + + def _init_discovery_test(self, tmp_path, mocker): + init = {} + init["src_dir"] = tmp_path + init["language"] = "PHP" + init["tool1"] = {"name": "dummyTool", "version": "1"} + init["tool2"] = {"name": "dummyTool", "version": "2"} + init["tool3"] = {"name": "anotherDummyTool", "version": "1.2"} + init["tp_lib_path"] = join_resources_path("sample_patlib") + mocked_tool_interface: Dict = { + "supported_languages": ["PHP"], + "tool_interface": "qualitytests.core.sast_test.SastTest" + } + mocker.patch("sast.utils.load_sast_specific_config", return_value=mocked_tool_interface) + mocker.patch("core.discovery.run_generate_cpg_cmd", + return_value="Done", + side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) + mocker.patch("core.discovery.run_joern_scala_query_for_test", + return_value="Done") + with open(join_resources_path("sample_joern") / "joern_discovery_query_res.json", "r") as data_file: + joern_res = json.load(data_file) + exp_discovery_rule_results = ( + join_resources_path("sample_joern") / "binary.bin", + "1_static_variables_iall", + joern_res + ) + mocker.patch("core.discovery.run_joern_discovery_rule", return_value=exp_discovery_rule_results) + return init + + + def test_run_discovery_for_pattern_list_1(self, tmp_path, capsys, mocker): + init = self._init_discovery_test(tmp_path, mocker) + itools: list[Dict] = [ + init["tool1"] + ] + # Test 1: two patterns with measurements + pattern_id_list: list[int] = [1, 2] + interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, + tp_lib_path=init["tp_lib_path"]) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and output_dir.is_dir() and output_dir.iterdir()) + + + def test_run_discovery_for_pattern_list_2(self, tmp_path, capsys, mocker): + init = self._init_discovery_test(tmp_path, mocker) + itools: list[Dict] = [ + init["tool1"] + ] + # Test 2: three patterns, one no measurements + pattern_id_list: list[int] = [1, 2, 3] + interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, + tp_lib_path=init["tp_lib_path"]) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and output_dir.is_dir() and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and in_logfile(logfile, measurementNotFound(3), lastNlines=2) + + + def test_run_discovery_for_pattern_list_3(self, tmp_path, capsys, mocker): + init = self._init_discovery_test(tmp_path, mocker) + itools: list[Dict] = [ + init["tool1"] + ] + # Test 3: two patterns with measurements, different output dir + pattern_id_list: list[int] = [1, 2] + ioutput_dir = join_resources_path("../temp").resolve() + ioutput_dir.mkdir(parents=True, exist_ok=True) + interface.run_discovery_for_pattern_list(init["src_dir"], pattern_id_list, init["language"], itools, + tp_lib_path=init["tp_lib_path"], output_dir=ioutput_dir) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and ioutput_dir == output_dir.parent and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and logfile.is_file() + + + def test_manual_discovery(self, tmp_path, capsys, mocker): + init = self._init_discovery_test(tmp_path, mocker) + src_dir: Path = tmp_path + discovery_method = "joern" + discovery_rule_list = [ + join_resources_path("sample_patlib/JS"), + join_resources_path("sample_patlib/PHP/1_static_variables"), + join_resources_path("sample_patlib/whatever"), + join_resources_path("sample_patlib/PHP/2_static_variables"), + "Whatever we want to put here that is not a proper folder :) \ $#$@!^&*()=+[]{}~`/", + join_resources_path("sample_patlib/PHP/3_global_array/1_instance_3_global_array/1_instance_3_global_array.sc") + ] + language = "PHP" + ioutput_dir = join_resources_path("../temp").resolve() + ioutput_dir.mkdir(parents=True, exist_ok=True) + interface.manual_discovery(src_dir, discovery_method, discovery_rule_list, language, + timeout_sec=0, output_dir=ioutput_dir) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and ioutput_dir == output_dir.parent and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and logfile.is_file() + + + # @pytest.mark.asyncio + # async def test_sast_measurement(self, tmp_path, capsys, mocker): + # init = {} + # init_measure_test(init, mocker) + # await interface.measure_list_patterns(init["patterns"], init["language"], tools=init["tools"], + # tp_lib_path=init["tp_lib_path"], output_dir=tmp_path, workers=2) + # out = capsys.readouterr().out + # captured_out_lines = out.split("\n") + # sys.stdout.write(out) + # output_dir = get_result_output_dir(captured_out_lines) + # assert (output_dir and output_dir.iterdir()) + # logfile = get_logfile_path(captured_out_lines) + # assert logfile and logfile.is_file() + # + # + # def test_sast_report_1(self, tmp_path, capsys, mocker): + # init = {} + # init_sastreport_test(init, mocker) + # # Test 1: it does not consider the following params + # # - export_file: Path = None, + # # - output_dir: Path = Path(config.RESULT_DIR).resolve(), + # # - only_last_measurement: bool = True): + # interface.report_sast_measurement_for_pattern_list( + # init["tools"], init["language"], init["patterns"], init["tp_lib_path"] + # ) + # out = capsys.readouterr().out + # captured_out_lines = out.split("\n") + # sys.stdout.write(out) + # output_dir = get_result_output_dir(captured_out_lines) + # assert (output_dir and output_dir.iterdir()) + # logfile = get_logfile_path(captured_out_lines) + # assert logfile and logfile.is_file() + # + # + # def test_sast_report_2(self, tmp_path, capsys, mocker): + # init = {} + # init_sastreport_test(init, mocker) + # export_file = "test_export.csv" + # # Test 2: it does not consider the following params + # # - only_last_measurement: bool = True): + # interface.report_sast_measurement_for_pattern_list( + # init["tools"], init["language"], init["patterns"], init["tp_lib_path"], + # export_file=export_file, output_dir=tmp_path + # ) + # out = capsys.readouterr().out + # captured_out_lines = out.split("\n") + # sys.stdout.write(out) + # output_dir = get_result_output_dir(captured_out_lines) + # assert (output_dir and output_dir.iterdir()) + # logfile = get_logfile_path(captured_out_lines) + # assert logfile and logfile.is_file() + # assert (output_dir / export_file).is_file() + # + # + # def test_check_discovery_rules_1(self, tmp_path, capsys, mocker): + # init = {} + # self._init_discovery_test(tmp_path, mocker) + # init_test(init) + # export_file = "test_export.csv" + # interface.check_discovery_rules( + # init["language"], init["patterns"], 0, + # tp_lib_path=init["tp_lib_path"], export_file=export_file + # ) + # out = capsys.readouterr().out + # captured_out_lines = out.split("\n") + # sys.stdout.write(out) + # output_dir = get_result_output_dir(captured_out_lines) + # assert (output_dir and output_dir.iterdir()) + # logfile = get_logfile_path(captured_out_lines) + # assert logfile and logfile.is_file() + # + # + # def test_check_discovery_rules_2(self, tmp_path, capsys, mocker): + # init = {} + # self._init_discovery_test(tmp_path, mocker) + # init_test(init) + # export_file = "test_export.csv" + # interface.check_discovery_rules( + # init["language"], init["patterns"], 0, + # tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path + # ) + # out = capsys.readouterr().out + # captured_out_lines = out.split("\n") + # sys.stdout.write(out) + # output_dir = get_result_output_dir(captured_out_lines) + # assert (output_dir and output_dir.iterdir()) + # logfile = get_logfile_path(captured_out_lines) + # assert logfile and logfile.is_file() + # + # + # def init_check_discovery_rules_3(self, init, tmp_path, mocker): + # init["language"] = "PHP" + # init["patterns"] = [32, 37] + # init["tp_lib_path"] = join_resources_path("sample_patlib_issue9") + # mocker.patch("core.discovery.run_generate_cpg_cmd", + # return_value="Done", + # side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) + # mocker.patch("core.discovery.run_joern_scala_query_for_test", + # return_value="Done") + # exp_discovery_rule_results = ( + # str(join_resources_path("sample_joern") / "binary.bin"), + # "1_static_variables_iall", + # 1 + # ) + # mocker.patch("core.discovery.run_and_process_discovery_rule", + # return_value=bytes(str(exp_discovery_rule_results), 'utf-8')) + # + # + # def test_check_discovery_rules_3(self, tmp_path, capsys, mocker): + # init = {} + # self.init_check_discovery_rules_3(init, tmp_path, mocker) + # export_file = "test_export.csv" + # interface.check_discovery_rules( + # init["language"], init["patterns"], 0, + # tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path + # ) + # out = capsys.readouterr().out + # captured_out_lines = out.split("\n") + # sys.stdout.write(out) + # output_dir = get_result_output_dir(captured_out_lines) + # assert (output_dir and output_dir.iterdir()) + # logfile = get_logfile_path(captured_out_lines) + # assert logfile and logfile.is_file() + # + # + # def test_repair_patterns_not_including_readme(self): + # sample_tp_lib = join_resources_path("sample_patlib") + # test_pattern = create_pattern() + # with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ + # patch("core.pattern.Pattern.repair") as patternrepair_mock, \ + # patch("core.utils.check_file_exist") as check_file_exists_mock, \ + # patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ + # patch("pathlib.Path.mkdir") as mkdir_mock: + # init_pattern_mock.return_value = test_pattern + # interface.repair_patterns("JS", [1,2,3], None, True, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) + # + # patternrepair_mock.assert_called_with(False, + # discovery_rule_results=Path("dr_results.csv"), + # measurement_results=Path("measurements"), + # masking_file=None) + # expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] + # init_pattern_mock.assert_has_calls(expected_calls) + # check_file_exists_mock.assert_not_called() + # measurement_result_exist_mock.assert_not_called() + # mkdir_mock.assert_called() + # + # def test_repair_patterns_not_including_readme(self): + # sample_tp_lib = join_resources_path("sample_patlib") + # test_pattern = create_pattern() + # with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ + # patch("core.pattern.Pattern.repair") as patternrepair_mock, \ + # patch("core.utils.check_file_exist") as check_file_exists_mock, \ + # patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ + # patch("pathlib.Path.mkdir") as mkdir_mock: + # init_pattern_mock.return_value = test_pattern + # interface.repair_patterns("JS", [1,2,3], None, False, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) + # + # patternrepair_mock.assert_called_with(True, + # discovery_rule_results=Path("dr_results.csv"), + # measurement_results=Path("measurements"), + # masking_file=None) + # expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] + # init_pattern_mock.assert_has_calls(expected_calls) + # check_file_exists_mock.assert_called() + # measurement_result_exist_mock.assert_called_once() + # mkdir_mock.assert_called() From 4f8f51597f320579b31ac51c1e30dacac1a00a7a Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Fri, 23 Feb 2024 15:06:11 +0100 Subject: [PATCH 16/30] Remove sast existing sast methods from tpframework and reimporting them from sast lib --- qualitytests/core/test_utils.py | 13 +++++++------ tp_framework/cli/tpf_commands.py | 4 ++-- tp_framework/core/discovery.py | 3 ++- tp_framework/core/errors.py | 13 ------------- tp_framework/core/exceptions.py | 27 --------------------------- tp_framework/core/measurement.py | 3 ++- tp_framework/core/utils.py | 4 ++-- 7 files changed, 15 insertions(+), 52 deletions(-) diff --git a/qualitytests/core/test_utils.py b/qualitytests/core/test_utils.py index 3a5e096..ad928d9 100644 --- a/qualitytests/core/test_utils.py +++ b/qualitytests/core/test_utils.py @@ -7,6 +7,7 @@ from core.exceptions import PatternDoesNotExists, TPLibDoesNotExist, LanguageTPLibDoesNotExist, DiscoveryMethodNotSupported from unittest.mock import patch, mock_open import qualitytests.qualitytests_utils as qualitytests_utils +import sast.utils as sast_utils def setup_three_pattern(tmp_path: Path): language: str = "PHP" @@ -101,12 +102,12 @@ def test_get_discovery_rules(self): def test_sast_tool_version_match(self): - assert utils.sast_tool_version_match("1.5.3", "1.5.3") - assert not utils.sast_tool_version_match("1.5.3", "1.5.2") - assert utils.sast_tool_version_match("1.5.3", "1.5.3.4", nv_max=3) - assert not utils.sast_tool_version_match("1.5.3", "1.5.3.4", nv_max=4) - assert not utils.sast_tool_version_match("1.5.3.4", "1.5.3", nv_max=4) - assert utils.sast_tool_version_match("1.5.3.4.5.6.", "1.5.3", nv_max=3) + assert sast_utils.sast_tool_version_match("1.5.3", "1.5.3") + assert not sast_utils.sast_tool_version_match("1.5.3", "1.5.2") + assert sast_utils.sast_tool_version_match("1.5.3", "1.5.3.4", nv_max=3) + assert not sast_utils.sast_tool_version_match("1.5.3", "1.5.3.4", nv_max=4) + assert not sast_utils.sast_tool_version_match("1.5.3.4", "1.5.3", nv_max=4) + assert sast_utils.sast_tool_version_match("1.5.3.4.5.6.", "1.5.3", nv_max=3) def test_get_pattern_dir_from_id(self): diff --git a/tp_framework/cli/tpf_commands.py b/tp_framework/cli/tpf_commands.py index e1674f9..06928a5 100644 --- a/tp_framework/cli/tpf_commands.py +++ b/tp_framework/cli/tpf_commands.py @@ -7,8 +7,8 @@ from cli import interface from core import utils -from core.exceptions import InvalidSastTools -from core.errors import invalidSastTools +from sast.exceptions import InvalidSastTools +from sast.errors import invalidSastTools from core.pattern import Pattern class Command(ABC): diff --git a/tp_framework/core/discovery.py b/tp_framework/core/discovery.py index 22fb526..3cca11c 100644 --- a/tp_framework/core/discovery.py +++ b/tp_framework/core/discovery.py @@ -14,12 +14,13 @@ import config from core import utils, measurement from core.exceptions import DiscoveryMethodNotSupported, MeasurementNotFound, CPGGenerationError, \ - CPGLanguageNotSupported, DiscoveryRuleError, DiscoveryRuleParsingResultError, InvalidSastTools + CPGLanguageNotSupported, DiscoveryRuleError, DiscoveryRuleParsingResultError from core.measurement import Measurement from core.instance import Instance from core.pattern import Pattern import sast.utils as sast_utils +from sast.exceptions import InvalidSastTools # mand_finding_joern_keys = ["filename", "methodFullName", "lineNumber"] mand_finding_joern_keys = ["filename", "lineNumber"] diff --git a/tp_framework/core/errors.py b/tp_framework/core/errors.py index bedfdf8..1a40c51 100644 --- a/tp_framework/core/errors.py +++ b/tp_framework/core/errors.py @@ -45,19 +45,6 @@ def languageTPLibDoesNotExist(): def targetDirDoesNotExist(): return "The target directory does not exists." - -def invalidSastTool(tool): - return f"SAST tool not found in the SAST yaml configuration: {tool}" - - -def invalidSastTools(): - return "Invalid SAST tools or none of them could be selected for the task." - - -def sastScanFailed(tool): - return f"SAST Scan failed for: {tool}." - - def discoveryMethodNotSupported(discovery_method): return f"Discovery method `{discovery_method}` is not supported." diff --git a/tp_framework/core/exceptions.py b/tp_framework/core/exceptions.py index c5af8f3..d63808a 100644 --- a/tp_framework/core/exceptions.py +++ b/tp_framework/core/exceptions.py @@ -64,33 +64,6 @@ def __init__(self, message=errors.targetDirDoesNotExist()): super().__init__(self.message) -class InvalidSastTool(Exception): - def __init__(self, tool, message=None): - if message: - self.message = message - else: - self.message = errors.invalidSastTool(tool) - super().__init__(self.message) - - -class InvalidSastTools(Exception): - def __init__(self, message=None, tool=None): - if message: - self.message = message - else: - self.message = errors.invalidSastTools() - super().__init__(self.message) - - -class SastScanFailed(Exception): - def __init__(self, message=None, tool=None): - if message: - self.message = message - else: - self.message = errors.sastScanFailed(tool) - super().__init__(self.message) - - class DiscoveryMethodNotSupported(Exception): def __init__(self, message=None, discovery_method=None): if message: diff --git a/tp_framework/core/measurement.py b/tp_framework/core/measurement.py index 4c142fa..935a119 100644 --- a/tp_framework/core/measurement.py +++ b/tp_framework/core/measurement.py @@ -4,6 +4,7 @@ import json from pathlib import Path from typing import Dict +import sast.utils as sast_utils import logging from core import loggermgr @@ -142,7 +143,7 @@ def load_last_measurement_for_tool(tool: Dict, language: str, tp_lib: Path, patt measurements_for_tool: list[Measurement] = list( filter(lambda m: m.tool == tool["name"] and - utils.sast_tool_version_match(m.version, tool["version"]), + sast_utils.sast_tool_version_match(m.version, tool["version"]), measurements) ) if not measurements_for_tool: diff --git a/tp_framework/core/utils.py b/tp_framework/core/utils.py index 6f15ff4..18662c8 100644 --- a/tp_framework/core/utils.py +++ b/tp_framework/core/utils.py @@ -16,8 +16,8 @@ logger = logging.getLogger(loggermgr.logger_name(__name__)) import config -from core.exceptions import PatternDoesNotExists, LanguageTPLibDoesNotExist, TPLibDoesNotExist, InvalidSastTools, \ - DiscoveryMethodNotSupported, TargetDirDoesNotExist, InvalidSastTool, InstanceDoesNotExists, \ +from core.exceptions import PatternDoesNotExists, LanguageTPLibDoesNotExist, TPLibDoesNotExist, \ + DiscoveryMethodNotSupported, TargetDirDoesNotExist, InstanceDoesNotExists, \ MeasurementResultsDoNotExist from core import errors From b9d5bae4edbf2ae539b1ade2fd4e6cd16408151d Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Tue, 27 Feb 2024 11:38:46 +0100 Subject: [PATCH 17/30] Fix Unhandled Cancellation Error in sast_task_runner --- tp_framework/core/sast_job_runner.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tp_framework/core/sast_job_runner.py b/tp_framework/core/sast_job_runner.py index 50fa758..3450312 100644 --- a/tp_framework/core/sast_job_runner.py +++ b/tp_framework/core/sast_job_runner.py @@ -32,7 +32,12 @@ async def sast_task_runner(name: str, in_queue: asyncio.Queue, out_queue: asynci try: csv_res = await task out_queue.put_nowait((job_id, tool_name, tool_version, instance, date, csv_res)) - except Exception as e: + # asyncio.CancelledError is a part of BaseException + # https://docs.python.org/3/library/asyncio-exceptions.html + # Note: if the Exceptions raised in a SAST JOB is not properly handled, it will stop the + # sast_task_runner (Worker), thus may end up in a situation where the out_queue is empty which will result + # in waiting indefinitely. + except (Exception, asyncio.CancelledError) as e: out_queue.put_nowait((job_id, tool_name, tool_version, instance, date, None)) logger.exception(f"{name} exception {e!r}") # raise From cbb40e54a6d47600ab54eef329dd57ea53ed56e0 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Tue, 27 Feb 2024 11:39:38 +0100 Subject: [PATCH 18/30] Uncomment tests --- qualitytests/cli/test_interface.py | 332 ++++++++++++++--------------- 1 file changed, 166 insertions(+), 166 deletions(-) diff --git a/qualitytests/cli/test_interface.py b/qualitytests/cli/test_interface.py index 00f3813..0ff535e 100644 --- a/qualitytests/cli/test_interface.py +++ b/qualitytests/cli/test_interface.py @@ -129,169 +129,169 @@ def test_manual_discovery(self, tmp_path, capsys, mocker): assert logfile and logfile.is_file() - # @pytest.mark.asyncio - # async def test_sast_measurement(self, tmp_path, capsys, mocker): - # init = {} - # init_measure_test(init, mocker) - # await interface.measure_list_patterns(init["patterns"], init["language"], tools=init["tools"], - # tp_lib_path=init["tp_lib_path"], output_dir=tmp_path, workers=2) - # out = capsys.readouterr().out - # captured_out_lines = out.split("\n") - # sys.stdout.write(out) - # output_dir = get_result_output_dir(captured_out_lines) - # assert (output_dir and output_dir.iterdir()) - # logfile = get_logfile_path(captured_out_lines) - # assert logfile and logfile.is_file() - # - # - # def test_sast_report_1(self, tmp_path, capsys, mocker): - # init = {} - # init_sastreport_test(init, mocker) - # # Test 1: it does not consider the following params - # # - export_file: Path = None, - # # - output_dir: Path = Path(config.RESULT_DIR).resolve(), - # # - only_last_measurement: bool = True): - # interface.report_sast_measurement_for_pattern_list( - # init["tools"], init["language"], init["patterns"], init["tp_lib_path"] - # ) - # out = capsys.readouterr().out - # captured_out_lines = out.split("\n") - # sys.stdout.write(out) - # output_dir = get_result_output_dir(captured_out_lines) - # assert (output_dir and output_dir.iterdir()) - # logfile = get_logfile_path(captured_out_lines) - # assert logfile and logfile.is_file() - # - # - # def test_sast_report_2(self, tmp_path, capsys, mocker): - # init = {} - # init_sastreport_test(init, mocker) - # export_file = "test_export.csv" - # # Test 2: it does not consider the following params - # # - only_last_measurement: bool = True): - # interface.report_sast_measurement_for_pattern_list( - # init["tools"], init["language"], init["patterns"], init["tp_lib_path"], - # export_file=export_file, output_dir=tmp_path - # ) - # out = capsys.readouterr().out - # captured_out_lines = out.split("\n") - # sys.stdout.write(out) - # output_dir = get_result_output_dir(captured_out_lines) - # assert (output_dir and output_dir.iterdir()) - # logfile = get_logfile_path(captured_out_lines) - # assert logfile and logfile.is_file() - # assert (output_dir / export_file).is_file() - # - # - # def test_check_discovery_rules_1(self, tmp_path, capsys, mocker): - # init = {} - # self._init_discovery_test(tmp_path, mocker) - # init_test(init) - # export_file = "test_export.csv" - # interface.check_discovery_rules( - # init["language"], init["patterns"], 0, - # tp_lib_path=init["tp_lib_path"], export_file=export_file - # ) - # out = capsys.readouterr().out - # captured_out_lines = out.split("\n") - # sys.stdout.write(out) - # output_dir = get_result_output_dir(captured_out_lines) - # assert (output_dir and output_dir.iterdir()) - # logfile = get_logfile_path(captured_out_lines) - # assert logfile and logfile.is_file() - # - # - # def test_check_discovery_rules_2(self, tmp_path, capsys, mocker): - # init = {} - # self._init_discovery_test(tmp_path, mocker) - # init_test(init) - # export_file = "test_export.csv" - # interface.check_discovery_rules( - # init["language"], init["patterns"], 0, - # tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path - # ) - # out = capsys.readouterr().out - # captured_out_lines = out.split("\n") - # sys.stdout.write(out) - # output_dir = get_result_output_dir(captured_out_lines) - # assert (output_dir and output_dir.iterdir()) - # logfile = get_logfile_path(captured_out_lines) - # assert logfile and logfile.is_file() - # - # - # def init_check_discovery_rules_3(self, init, tmp_path, mocker): - # init["language"] = "PHP" - # init["patterns"] = [32, 37] - # init["tp_lib_path"] = join_resources_path("sample_patlib_issue9") - # mocker.patch("core.discovery.run_generate_cpg_cmd", - # return_value="Done", - # side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) - # mocker.patch("core.discovery.run_joern_scala_query_for_test", - # return_value="Done") - # exp_discovery_rule_results = ( - # str(join_resources_path("sample_joern") / "binary.bin"), - # "1_static_variables_iall", - # 1 - # ) - # mocker.patch("core.discovery.run_and_process_discovery_rule", - # return_value=bytes(str(exp_discovery_rule_results), 'utf-8')) - # - # - # def test_check_discovery_rules_3(self, tmp_path, capsys, mocker): - # init = {} - # self.init_check_discovery_rules_3(init, tmp_path, mocker) - # export_file = "test_export.csv" - # interface.check_discovery_rules( - # init["language"], init["patterns"], 0, - # tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path - # ) - # out = capsys.readouterr().out - # captured_out_lines = out.split("\n") - # sys.stdout.write(out) - # output_dir = get_result_output_dir(captured_out_lines) - # assert (output_dir and output_dir.iterdir()) - # logfile = get_logfile_path(captured_out_lines) - # assert logfile and logfile.is_file() - # - # - # def test_repair_patterns_not_including_readme(self): - # sample_tp_lib = join_resources_path("sample_patlib") - # test_pattern = create_pattern() - # with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ - # patch("core.pattern.Pattern.repair") as patternrepair_mock, \ - # patch("core.utils.check_file_exist") as check_file_exists_mock, \ - # patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ - # patch("pathlib.Path.mkdir") as mkdir_mock: - # init_pattern_mock.return_value = test_pattern - # interface.repair_patterns("JS", [1,2,3], None, True, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) - # - # patternrepair_mock.assert_called_with(False, - # discovery_rule_results=Path("dr_results.csv"), - # measurement_results=Path("measurements"), - # masking_file=None) - # expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] - # init_pattern_mock.assert_has_calls(expected_calls) - # check_file_exists_mock.assert_not_called() - # measurement_result_exist_mock.assert_not_called() - # mkdir_mock.assert_called() - # - # def test_repair_patterns_not_including_readme(self): - # sample_tp_lib = join_resources_path("sample_patlib") - # test_pattern = create_pattern() - # with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ - # patch("core.pattern.Pattern.repair") as patternrepair_mock, \ - # patch("core.utils.check_file_exist") as check_file_exists_mock, \ - # patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ - # patch("pathlib.Path.mkdir") as mkdir_mock: - # init_pattern_mock.return_value = test_pattern - # interface.repair_patterns("JS", [1,2,3], None, False, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) - # - # patternrepair_mock.assert_called_with(True, - # discovery_rule_results=Path("dr_results.csv"), - # measurement_results=Path("measurements"), - # masking_file=None) - # expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] - # init_pattern_mock.assert_has_calls(expected_calls) - # check_file_exists_mock.assert_called() - # measurement_result_exist_mock.assert_called_once() - # mkdir_mock.assert_called() + @pytest.mark.asyncio + async def test_sast_measurement(self, tmp_path, capsys, mocker): + init = {} + init_measure_test(init, mocker) + await interface.measure_list_patterns(init["patterns"], init["language"], tools=init["tools"], + tp_lib_path=init["tp_lib_path"], output_dir=tmp_path, workers=2) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and logfile.is_file() + + + def test_sast_report_1(self, tmp_path, capsys, mocker): + init = {} + init_sastreport_test(init, mocker) + # Test 1: it does not consider the following params + # - export_file: Path = None, + # - output_dir: Path = Path(config.RESULT_DIR).resolve(), + # - only_last_measurement: bool = True): + interface.report_sast_measurement_for_pattern_list( + init["tools"], init["language"], init["patterns"], init["tp_lib_path"] + ) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and logfile.is_file() + + + def test_sast_report_2(self, tmp_path, capsys, mocker): + init = {} + init_sastreport_test(init, mocker) + export_file = "test_export.csv" + # Test 2: it does not consider the following params + # - only_last_measurement: bool = True): + interface.report_sast_measurement_for_pattern_list( + init["tools"], init["language"], init["patterns"], init["tp_lib_path"], + export_file=export_file, output_dir=tmp_path + ) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and logfile.is_file() + assert (output_dir / export_file).is_file() + + + def test_check_discovery_rules_1(self, tmp_path, capsys, mocker): + init = {} + self._init_discovery_test(tmp_path, mocker) + init_test(init) + export_file = "test_export.csv" + interface.check_discovery_rules( + init["language"], init["patterns"], 0, + tp_lib_path=init["tp_lib_path"], export_file=export_file + ) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and logfile.is_file() + + + def test_check_discovery_rules_2(self, tmp_path, capsys, mocker): + init = {} + self._init_discovery_test(tmp_path, mocker) + init_test(init) + export_file = "test_export.csv" + interface.check_discovery_rules( + init["language"], init["patterns"], 0, + tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path + ) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and logfile.is_file() + + + def init_check_discovery_rules_3(self, init, tmp_path, mocker): + init["language"] = "PHP" + init["patterns"] = [32, 37] + init["tp_lib_path"] = join_resources_path("sample_patlib_issue9") + mocker.patch("core.discovery.run_generate_cpg_cmd", + return_value="Done", + side_effect=create_mock_cpg(tmp_path / "cpg_binary.bin")) + mocker.patch("core.discovery.run_joern_scala_query_for_test", + return_value="Done") + exp_discovery_rule_results = ( + str(join_resources_path("sample_joern") / "binary.bin"), + "1_static_variables_iall", + 1 + ) + mocker.patch("core.discovery.run_and_process_discovery_rule", + return_value=bytes(str(exp_discovery_rule_results), 'utf-8')) + + + def test_check_discovery_rules_3(self, tmp_path, capsys, mocker): + init = {} + self.init_check_discovery_rules_3(init, tmp_path, mocker) + export_file = "test_export.csv" + interface.check_discovery_rules( + init["language"], init["patterns"], 0, + tp_lib_path=init["tp_lib_path"], export_file=export_file, output_dir=tmp_path + ) + out = capsys.readouterr().out + captured_out_lines = out.split("\n") + sys.stdout.write(out) + output_dir = get_result_output_dir(captured_out_lines) + assert (output_dir and output_dir.iterdir()) + logfile = get_logfile_path(captured_out_lines) + assert logfile and logfile.is_file() + + + def test_repair_patterns_not_including_readme(self): + sample_tp_lib = join_resources_path("sample_patlib") + test_pattern = create_pattern() + with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ + patch("core.pattern.Pattern.repair") as patternrepair_mock, \ + patch("core.utils.check_file_exist") as check_file_exists_mock, \ + patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ + patch("pathlib.Path.mkdir") as mkdir_mock: + init_pattern_mock.return_value = test_pattern + interface.repair_patterns("JS", [1,2,3], None, True, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) + + patternrepair_mock.assert_called_with(False, + discovery_rule_results=Path("dr_results.csv"), + measurement_results=Path("measurements"), + masking_file=None) + expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] + init_pattern_mock.assert_has_calls(expected_calls) + check_file_exists_mock.assert_not_called() + measurement_result_exist_mock.assert_not_called() + mkdir_mock.assert_called() + + def test_repair_patterns_not_including_readme(self): + sample_tp_lib = join_resources_path("sample_patlib") + test_pattern = create_pattern() + with patch("core.pattern.Pattern.init_from_id_and_language") as init_pattern_mock, \ + patch("core.pattern.Pattern.repair") as patternrepair_mock, \ + patch("core.utils.check_file_exist") as check_file_exists_mock, \ + patch("core.utils.check_measurement_results_exist") as measurement_result_exist_mock, \ + patch("pathlib.Path.mkdir") as mkdir_mock: + init_pattern_mock.return_value = test_pattern + interface.repair_patterns("JS", [1,2,3], None, False, Path("measurements"), Path("dr_results.csv"), Path("out"), sample_tp_lib) + + patternrepair_mock.assert_called_with(True, + discovery_rule_results=Path("dr_results.csv"), + measurement_results=Path("measurements"), + masking_file=None) + expected_calls = [call(1, "JS", sample_tp_lib), call(2, "JS", sample_tp_lib), call(3, "JS", sample_tp_lib)] + init_pattern_mock.assert_has_calls(expected_calls) + check_file_exists_mock.assert_called() + measurement_result_exist_mock.assert_called_once() + mkdir_mock.assert_called() From d1c7f74036d67cc64bd415decff96e13dada4866 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Wed, 28 Feb 2024 08:55:50 +0100 Subject: [PATCH 19/30] Remove sast from submodule --- .gitmodules | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 22d5a1f..3a2c4d9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,4 @@ [submodule "testability_patterns"] path = testability_patterns url = git@github.com:testable-eu/sast-testability-patterns.git -[submodule "sast"] - path = sast - url = git@github.com:sacumesh/sast.git - branch = feature/sast-extraction From e083252236d1e6584ac89b96ea00e430ad19fee4 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Wed, 28 Feb 2024 08:56:54 +0100 Subject: [PATCH 20/30] Remove sast cached --- sast | 1 - 1 file changed, 1 deletion(-) delete mode 160000 sast diff --git a/sast b/sast deleted file mode 160000 index 4502bf6..0000000 --- a/sast +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4502bf671967b0fa716174a9a7822efa15728f9c From 423ab8eef534b42d947fdd83ae024cb10758ba92 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Wed, 28 Feb 2024 09:03:36 +0100 Subject: [PATCH 21/30] Add submodule SAST --- .gitmodules | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitmodules b/.gitmodules index 3a2c4d9..0673955 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = testability_patterns url = git@github.com:testable-eu/sast-testability-patterns.git +[submodule "SAST"] + path = SAST + url = git@github.com:sacumesh/SAST.git + branch = feature/sast-extraction From 90250f8a23e1b49a8df0e2324e962411d950474b Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Wed, 28 Feb 2024 09:06:53 +0100 Subject: [PATCH 22/30] Update SAST --- SAST | 1 + 1 file changed, 1 insertion(+) create mode 160000 SAST diff --git a/SAST b/SAST new file mode 160000 index 0000000..d38318a --- /dev/null +++ b/SAST @@ -0,0 +1 @@ +Subproject commit d38318a708ae9c58b40ee9775951621b2bd906ef From fb768ad7df57a4926c769a9e59a053dc324998ae Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Wed, 28 Feb 2024 09:20:55 +0100 Subject: [PATCH 23/30] Fix SAST dir path --- Dockerfile | 2 +- docker-compose-dev.yml | 2 +- docker-compose.yml | 4 +--- requirements-dev.txt | Bin 342 -> 342 bytes requirements.txt | Bin 260 -> 260 bytes 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1fc43e2..a53b13f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ COPY config.py ${TPF_HOME}/config.py COPY setup.py ${TPF_HOME}/setup.py COPY pytest.ini ${TPF_HOME}/pytest.ini -COPY sast ${TPF_HOME}/sast +COPY SAST ${TPF_HOME}/SAST ARG TESTS_DIR="qualitytests" COPY ${TESTS_DIR}/requirements.txt ${TPF_HOME}/${TESTS_DIR}/requirements.txt diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index c042855..dd67c40 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -5,7 +5,7 @@ services: codeql: image: tpf_codeql build: - context: './sast/sast/codeql' + context: './SAST/sast/codeql' args: HOME: 'tp-framework' dockerfile: "./Dockerfile" diff --git a/docker-compose.yml b/docker-compose.yml index da0403f..84cad4c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,12 +5,11 @@ services: codeql: image: tpf_codeql build: - context: './sast/sast/codeql' + context: './SAST/sast/codeql' args: HOME: 'tp-framework' dockerfile: "./Dockerfile" volumes: - - codeql_interface:/tp-framework/SAST/codeql - codeql:/codeql tp-framework: @@ -22,7 +21,6 @@ services: env_file: - ./.env volumes: - - codeql_interface:/tp-framework/SAST/codeql - codeql:/codeql - ./testability_patterns:/tp-framework/testability_patterns - ./out:/tp-framework/out diff --git a/requirements-dev.txt b/requirements-dev.txt index fd7b1363d55496b5b49d83c71c5e5dcccd9d9262..474164551c72c1e7d162efefa97c89b5ae5cd0b4 100644 GIT binary patch delta 28 ccmcb{bd7032zxMtBSSDl$i!%WICJAt0Cq13I{*Lx delta 28 ccmcb{bd7032zxO@B117l$;4=XICJAt0D`XxI{*Lx diff --git a/requirements.txt b/requirements.txt index 42235900aa98d8e569ce12dae0d5da1e1d04481a..e8552897162406ea59b39911ceb5ad8aa79c3edc 100644 GIT binary patch delta 28 ccmZo+YGIlX#2(Dx$PmmBGBMH<&RiJ{0Au+FI{*Lx delta 28 ccmZo+YGIlX#9qvh$WY8sGBMH<&RiJ{0C0H-I{*Lx From 75c649c76e1ce25b9bc71c4e45b9d15d096f2f50 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Fri, 1 Mar 2024 16:28:02 +0100 Subject: [PATCH 24/30] Edit docker files to sast.py --- Dockerfile | 7 +++++-- requirements-dev.txt | Bin 342 -> 242 bytes requirements.txt | Bin 260 -> 166 bytes 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a53b13f..ea68282 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,12 +22,14 @@ ARG DISCOVERY_HOME="${TPF_HOME}/discovery" COPY tp_framework ${TPF_HOME}/tp_framework COPY testability_patterns ${TPF_HOME}/testability_patterns +COPY SAST/sast /SAST/sast +COPY SAST/sast.py ${TPF_HOME}/ +COPY SAST/requirements.txt ${TPF_HOME}/SAST/ + COPY config.py ${TPF_HOME}/config.py COPY setup.py ${TPF_HOME}/setup.py COPY pytest.ini ${TPF_HOME}/pytest.ini -COPY SAST ${TPF_HOME}/SAST - ARG TESTS_DIR="qualitytests" COPY ${TESTS_DIR}/requirements.txt ${TPF_HOME}/${TESTS_DIR}/requirements.txt @@ -44,6 +46,7 @@ RUN /bin/sh -c 'cd ${DISCOVERY_HOME}/joern/ && ./joern-install.sh --version=${JO # ADD HERE COMMANDS USEFUL FOR OTHER DOCKER-COMPOSE SERVICES ENV PYTHONPATH "${PYTHONPATH}:${TPF_HOME}/tp_framework" +ENV PYTHONPATH "${PYTHONPATH}:/SAST" RUN python setup.py develop diff --git a/requirements-dev.txt b/requirements-dev.txt index 474164551c72c1e7d162efefa97c89b5ae5cd0b4..868afd40310221ef2717688f952a9b5549e4180a 100644 GIT binary patch delta 9 Qcmcb{^oemo+r(En02dwv$p8QV delta 101 zcmeywc#Ua7TeLER0)r<*9z!uh2}2@74iG8;*@<92F9R2Y9)mtZFoPoyhX7?jssn(s kB@7urRtZBsn9N{E1=7U~r3^_7xj?ZLAS;IhS(*OVf delta 99 zcmZ3+*upfSAySz^fx(j@kD-{Mgdve32M85_>_jl1i$RY;pCOpR5r{*8;vlsFK)Diz i3?Qq7As Date: Fri, 1 Mar 2024 18:23:20 +0100 Subject: [PATCH 25/30] Update docker file --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ea68282..e0d9ff7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,6 @@ COPY tp_framework ${TPF_HOME}/tp_framework COPY testability_patterns ${TPF_HOME}/testability_patterns COPY SAST/sast /SAST/sast -COPY SAST/sast.py ${TPF_HOME}/ COPY SAST/requirements.txt ${TPF_HOME}/SAST/ COPY config.py ${TPF_HOME}/config.py From 46de1ddf2a1604ed3cd5f55fa8d9244eb55cf034 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Sun, 3 Mar 2024 13:55:50 +0100 Subject: [PATCH 26/30] Update Sast Module --- SAST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SAST b/SAST index d38318a..d3f12b9 160000 --- a/SAST +++ b/SAST @@ -1 +1 @@ -Subproject commit d38318a708ae9c58b40ee9775951621b2bd906ef +Subproject commit d3f12b999d9bcacf3ad2171195ee856e421aefd3 From 1536d363155430e52ec23a18de851a5906767dcd Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Sun, 3 Mar 2024 14:16:30 +0100 Subject: [PATCH 27/30] Update codeql HOME --- docker-compose-dev.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index dd67c40..ef5960e 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -7,7 +7,7 @@ services: build: context: './SAST/sast/codeql' args: - HOME: 'tp-framework' + HOME: '/SAST' dockerfile: "./Dockerfile" volumes: - codeql:/codeql diff --git a/docker-compose.yml b/docker-compose.yml index 84cad4c..2d4f912 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: build: context: './SAST/sast/codeql' args: - HOME: 'tp-framework' + HOME: '/SAST' dockerfile: "./Dockerfile" volumes: - codeql:/codeql From eeed815b3fb54782e76b3c331ca84bf4fcbe6e74 Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Mon, 4 Mar 2024 10:05:47 +0100 Subject: [PATCH 28/30] Add SAST_HOME arg --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e0d9ff7..434bb7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,9 @@ ARG DISCOVERY_HOME="${TPF_HOME}/discovery" COPY tp_framework ${TPF_HOME}/tp_framework COPY testability_patterns ${TPF_HOME}/testability_patterns -COPY SAST/sast /SAST/sast -COPY SAST/requirements.txt ${TPF_HOME}/SAST/ +ARG SAST_HOME="/SAST" +COPY SAST/sast ${SAST_HOME}/sast +COPY SAST/requirements.txt ${TPF_HOME}/${SAST_HOME}/ COPY config.py ${TPF_HOME}/config.py COPY setup.py ${TPF_HOME}/setup.py @@ -45,7 +46,7 @@ RUN /bin/sh -c 'cd ${DISCOVERY_HOME}/joern/ && ./joern-install.sh --version=${JO # ADD HERE COMMANDS USEFUL FOR OTHER DOCKER-COMPOSE SERVICES ENV PYTHONPATH "${PYTHONPATH}:${TPF_HOME}/tp_framework" -ENV PYTHONPATH "${PYTHONPATH}:/SAST" +ENV PYTHONPATH "${PYTHONPATH}:${SAST_HOME}" RUN python setup.py develop From 756e66935233949ff83d9efc54b9d1692337467f Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Mon, 4 Mar 2024 10:06:02 +0100 Subject: [PATCH 29/30] Update readme --- docs/How-to-add-a-SAST-tool.md | 208 ++------------------------------- 1 file changed, 9 insertions(+), 199 deletions(-) diff --git a/docs/How-to-add-a-SAST-tool.md b/docs/How-to-add-a-SAST-tool.md index fe2635d..7d05761 100644 --- a/docs/How-to-add-a-SAST-tool.md +++ b/docs/How-to-add-a-SAST-tool.md @@ -1,184 +1,20 @@ # How to: Add a SAST tool -The framework uses the arsenal of all the SAST tools provided in the `SAST/` folder. +The framework uses the arsenal of all the SAST tools provided in the `SAST` module. To add a SAST tools to your arsenal, please follows the steps listed hereafter and detailed in the chapters below: -[**1. File system structure**](#1-file-system-structure): comply with the recommended file system structure +[**1. Add sast tool the SAST module**](https://github.com/testable-eu/SAST/blob/main/README.md): Add the tool here. -[**2. Config files**](#2-config-files): prepare the config files +[**2. Docker**](#2-docker): create a specific dockerfile and add a service to the docker-compose -[**3. Implement the SAST interface**](#3-implement-the-sast-interface): python interface of the `tp_framework/core/sast.py` needs to be implemented - -[**4. Docker**](#4-docker): create a specific dockerfile and add a service to the docker-compose - -We illustrate all the steps hereafter by using the CodeQL v2.9.2 SAST tool as example. - -> At the time of writing, CodeQL is free for research and open source, https://codeql.github.com/ - -## 1. File system structure - -The following file system structure is recommended: - -```text -|__SAST - |__sast-config.yaml - |__codeql - | |__codeql_v2.9.2 - | | |__resources - | | |__config.yaml - | | |__codeql.py - | | |__Dockerfile - | | - | |__codeql_vX.Y.Z - | |__... - | - |__another_sast_tool - |__resources - |__config.yaml - ... -``` - -## 2. Config files - -### Main config file - -Each SAST tool needs to be declared in the main `SAST/sast-config.yaml` - -```yml -tools: - codeql: - version: - 2.9.2: - config: "./codeql/codeql_v2.9.2/config.yaml" - deploy: true - X.Y.Z: - ... - another_sast_tool: - version: - saas: - config: "./another_sast_tool/config.yaml" - deploy: false -``` - -In the above, an entry for CodeQL v2.9.2 is provided that specifies: - -- the config file for that tool is `./codeql/codeql_v2.9.2/config.yaml` -- that tool should be deployed. - -Some SAST tools are only available as SaaS so that is not possible to use multiple versions for them. The keyword `saas` as version specified that the tool is served as SaaS. When `deploy` is set to `false`, the tool will be neither deployed nor used within the framework. - -### SAST tool config file - -Each SAST tool will need to have its own `config.yaml` as follows: - -```yaml -name: "codeql" -version: "2.9.2" -supported_languages: - - "JS" - - "JAVA" - - "PYTHON" - - "RUBY" - - "CPP" - - "CSHARP" - - "GO" -tool_interface: "SAST.codeql.codeql_v2_9_2.codeql.CodeQL_v_2_9_2" -supported_vulnerability: - xss: "xss" - -# Specific SAST config parameters -installation_path: "/codeql" -``` - -Most of the fields are self-explanatory. Hereafter some details for the others: - -- `tool_interface` specifies where is the class instance implementing the SAST tool interface. In this specific case the class `CodeQL_v_2_9_2` is declared in the file `codeql.py` that is in the folder `./SAST.codeql.codeql_v2_9_2/` -- `supported_vulnerability` specifies the mapping between the vulnerabilities targeted by our framework and the way these vulnerabilities are represented within the SAST tool. In the specific example, the `xss` vulnerability is represented with the string `"xss"` in codeql, but other tools represent that with e.g., `"Cross-Site Scripting"`. -- `installation_path` indicates the path where the SAST tool will be installed on the docker container. - -## 3. Implement the SAST interface - -The python interface of the `SAST` class in `tp_framework/core/sast.py` needs to be implemented. This mainly requires the implementation of the following methods: - -- `launcher`: script to scan an application with the SAST tool to add, and -- `inspector`: script to inspect the findings from the SAST tool to add. - -### Launcher - -The launcher runs the SAST analysis and needs to implement this abstract method: - -```python -@abc.abstractmethod -async def launcher(self, src_dir: Path, language: str, **kwargs) -> Path: - raise NotImplementedError -``` - -Inputs: - -- `src_dir` is the path to the project to be scanned with SAST -- `language` is the programming language targeted by the SAST analysis -- `**kwargs` are specific arguments that could be used by the specific SAST tool - -Output: - -- path to the result file that will be then inspected - -## Inspector - -The inspector inspects the output of the SAST analysis to make it available to the tp-framework in a precise normalized format. It implements the following: - -```python -@abc.abstractmethod -def inspector(self, sast_res_file: Path, language: str) -> list[Dict]: - raise NotImplementedError -``` - -Inputs: - -- `sast_res_file`: path to the result file obtained by a launched scan -- `language` is the programming language targeted by the SAST analysis - -Output: - -- a list of dict entries complying with the format hereafter: - -```list -[ - { - type: "xss", - file: "foo.php", - line: "15" - }, - { - type: "sqli", - file: "whatever.php", - line: "169" - }, - -... -] -``` - -The first entry in the list above specifies that an cross-site scripting (`xss`) finding is reported for file `foo.php` at line `15` (sink of the `xss`) - -The types of vulnerabilities (`type`) currently supported by our framework are: -| normalized type | natural language | -|-------------------|-----------------------| -| xss | Cross-Site Scripting | -| sqli | SQL Injection | -| command_injection | Command Injection | -| path_manipulation | Path Manipulation | - -## 4. Docker +## 2. Docker The entire tp-framework is based on docker-compose. The installation of a SAST tool is done as a docker-compose service that invokes a Dockerfile. As such the dockerization of a new SAST tool requires the following sub-steps: **4.1. docker-compose files**: edit the main `docker-compose` files -**4.2. Dockerfile**: create the `Dockerfile` for the SAST tool - -**4.3. Composition/sharing corner-cases**: we experienced some corner-cases situations whose solution can be helpful for you as well +**4.2. Composition/sharing corner-cases**: we experienced some corner-cases situations whose solution can be helpful for you as well ### 4.1. docker-compose files @@ -191,11 +27,12 @@ services: codeql: # ADDED: NEW SERVICE FOR SAST TOOL image: tpf_codeql build: - context: './SAST/codeql/codeql_v2_9_2' + context: './SAST/sast/codeql/codeql_v2_9_2' dockerfile: "./Dockerfile" + args: + HOME: '/SAST' volumes: - codeql_v2_9_2:/codeql - - codeql_interface_v2_9_2:/tp-framework/SAST/codeql/codeql_v2_9_2 tp-framework: build: @@ -208,7 +45,6 @@ services: - ./.env volumes: - codeql_v2_9_2:/codeql # ADDED: SAST TOOL VOLUME USED BY tp-framework SERVICE - - codeql_interface_v2_9_2:/tp-framework/SAST/codeql/codeql_v2_9_2 # ADDED: " - ./testability_patterns:/tp-framework/testability_patterns - ./out:/tp-framework/out - ./in:/tp-framework/in @@ -216,39 +52,13 @@ services: volumes: codeql_v2_9_2: # ADDED - codeql_interface_v2_9_2: # ADDED ``` An `./.env` file is available for environment variables. This can be useful to solve some of the corner-cases we experienced (see section below). Also notice that the same additions will need to be migrated into the `docker-compose-dev.yml` file (this may be automated one day). -### 4.2. Dockerfile - -The Dockerfile shall specify the operations to execute in order to install the SAST tool and make it available for the tp-framework. Hereafter the example for Codeql. - -```dockerfile -FROM adoptopenjdk/openjdk11 - -ARG TPF_HOME="/tp-framework" -ARG CODEQL_INTERFACE_DIR="${TPF_HOME}/SAST/codeql/codeql_v2_9_2" - -COPY ./__init__.py ${CODEQL_INTERFACE_DIR}/__init__.py -COPY ./config.yaml ${CODEQL_INTERFACE_DIR}/config.yaml -COPY ./codeql.py ${CODEQL_INTERFACE_DIR}/codeql.py -COPY ./resources ${CODEQL_INTERFACE_DIR}/resources - -RUN apt-get update -RUN apt-get install wget - -RUN wget https://github.com/github/codeql-action/releases/download/codeql-bundle-20220512/codeql-bundle-linux64.tar.gz -RUN tar -xvzf ./codeql-bundle-linux64.tar.gz - -RUN ./codeql/codeql resolve languages -RUN ./codeql/codeql resolve qlpacks -``` - -### 4.3. Composition/sharing corner-cases +### 4.2. Composition/sharing corner-cases We experienced some corner-cases situations whose solutions can be helpful for you as well. Here some cases: From 6c951cb34f12de68ade30afee3b8914af1b688af Mon Sep 17 00:00:00 2001 From: Sachiththa Bandaranayake Date: Mon, 4 Mar 2024 11:10:06 +0100 Subject: [PATCH 30/30] Update SAST module --- SAST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SAST b/SAST index d3f12b9..8d278d1 160000 --- a/SAST +++ b/SAST @@ -1 +1 @@ -Subproject commit d3f12b999d9bcacf3ad2171195ee856e421aefd3 +Subproject commit 8d278d102a0a12d6de090b96b8f8891b51876538