diff --git a/tests/fixtures/arn.xml b/tests/fixtures/arn.xml new file mode 100644 index 0000000..b9ba3ec --- /dev/null +++ b/tests/fixtures/arn.xml @@ -0,0 +1,12 @@ + + + + aws:arn: + arn:aws:kms:{REGION}:{ACCOUNT}:key/{KEY_ID} + + + arn:aws:kms:eu-central-1:123456123456:key/hardcoded + arn:aws:kms:ap-southeast-1:123456123456:key/hardcoded + arn:aws:iam::123456123456:oidc-provider/auth-dev.mozilla.auth0.com + + diff --git a/tests/fixtures/arn.yml b/tests/fixtures/arn.yml new file mode 100644 index 0000000..71ec09e --- /dev/null +++ b/tests/fixtures/arn.yml @@ -0,0 +1,11 @@ +compliant: + arn01: "aws:arn:" + arn02: arn:aws:kms:{REGION}:{ACCOUNT}:key/{KEY_ID} + + +noncompliant: + arn01: arn:aws:kms:eu-central-1:123456123456:key/hardcoded + arn02: arn:aws:kms:ap-southeast-1:123456123456:key/hardcoded + arn03: arn:aws:iam::123456123456:oidc-provider/auth-dev.mozilla.auth0.com + arn_list: + - arn:aws:kms:eu-central-1:123456123456:key/hardcoded diff --git a/tests/fixtures/private-pgp-block.txt b/tests/fixtures/private-pgp-block.txt index f5fd58a..03fc4ab 100644 --- a/tests/fixtures/private-pgp-block.txt +++ b/tests/fixtures/private-pgp-block.txt @@ -2,6 +2,7 @@ Compliant: -----ok ---ok--- +-----ok----- Noncompliant: diff --git a/tests/fixtures/sops.yml b/tests/fixtures/sops.yml new file mode 100644 index 0000000..281e065 --- /dev/null +++ b/tests/fixtures/sops.yml @@ -0,0 +1,7 @@ +# https://github.com/mozilla/sops + +compliant: + password: ENC[AES256_GCM,data:HARDCODED,iv:1=,aad:No=,tag:k=] + +noncompliant: + password: hardcoded01 diff --git a/tests/unit/core/test_pairs.py b/tests/unit/core/test_pairs.py index b7fe6b7..379ca17 100644 --- a/tests/unit/core/test_pairs.py +++ b/tests/unit/core/test_pairs.py @@ -6,7 +6,7 @@ from tests.unit.conftest import FIXTURE_PATH, config_path, fixture_path, forbidden_path, tmp_path from whispers.core.args import parse_args from whispers.core.config import load_config -from whispers.core.pairs import filter_included, filter_static, is_static, load_plugin, make_pairs, tag_file +from whispers.core.pairs import filter_included, filter_static, load_plugin, make_pairs, tag_file from whispers.models.pair import KeyValuePair from whispers.plugins.config import Config from whispers.plugins.dockercfg import Dockercfg @@ -98,37 +98,6 @@ def test_filter_static(key, value, expected): assert filter_static(pair) == expected -@pytest.mark.parametrize( - ("key", "value", "expected"), - [ - (None, None, False), - ("key", "", False), - ("key", "$value", False), - ("key", "{{value}}", False), - ("key", "{value}", False), - ("key", "{whispers~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~}", False), - ("key", "{d2hpc3BlcnN+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+}", True), - ("key", "${value$}", False), - ("key", "", False), - ("key", "{value}", False), - ("key", "null", False), - ("key", "!Ref Value", False), - ("key", "{value}", False), - ("key", "/system/path/value", False), - ("thesame", "THESAME", False), - ("label", "WhispersLabel", False), - ("_key", "-key", False), - ("_secret_value_placeholder_", "----SECRET-VALUE-PLACEHOLDER-", False), - ("_secret_value_placeholder_", "----SECRET-VALUE-PLACEHOLDER--", True), - ("SECRET_VALUE_KEY", "whispers", True), - ("whispers", "SECRET_VALUE_PLACEHOLDER", True), - ("secret", "whispers", True), - ], -) -def test_is_static(key, value, expected): - assert is_static(key, value) == expected - - @pytest.mark.parametrize( ("filename", "expected"), [ diff --git a/tests/unit/core/test_secrets.py b/tests/unit/core/test_secrets.py index 882a53c..1c5d5b5 100644 --- a/tests/unit/core/test_secrets.py +++ b/tests/unit/core/test_secrets.py @@ -34,7 +34,7 @@ def test_filter_rule_lineno(rule_fixture): ], ) def test_detect_secrets_by_key(src, expected): - args = parse_args([fixture_path(src)]) + args = parse_args(["-S", "MINOR", fixture_path(src)]) config = load_config(args) rules = load_rules(args, config) pairs = make_pairs(config, FIXTURE_PATH.joinpath(src)) @@ -54,6 +54,8 @@ def test_detect_secrets_by_key(src, expected): ("apikeys.json", "MAJOR", 9), ("apikeys.xml", "MAJOR", 9), ("apikeys.yml", "MAJOR", 9), + ("arn.yml", "MINOR", 4), + ("arn.xml", "MINOR", 3), ("aws.yml", "BLOCKER", 3), ("aws.json", "BLOCKER", 3), ("aws.xml", "BLOCKER", 3), @@ -118,6 +120,7 @@ def test_detect_secrets_by_key(src, expected): ("settings01.ini", "CRITICAL", 1), ("settings02.ini", "CRITICAL", 1), ("severity.yml", "BLOCKER", 1), + ("sops.yml", DEFAULT_SEVERITY, 1), ("uri.yml", "CRITICAL", 3), ("webhooks.yml", "MINOR", 6), ], diff --git a/tests/unit/core/test_utils.py b/tests/unit/core/test_utils.py index ddac705..8e32355 100644 --- a/tests/unit/core/test_utils.py +++ b/tests/unit/core/test_utils.py @@ -16,6 +16,7 @@ is_iac, is_luhn, is_path, + is_static, is_uri, list_rule_prop, load_regex, @@ -123,6 +124,37 @@ def test_load_yaml_from_file(configfile, expected, raised): assert result == expected +@pytest.mark.parametrize( + ("key", "value", "expected"), + [ + (None, None, False), + ("key", "", False), + ("key", "$value", False), + ("key", "{{value}}", False), + ("key", "{value}", False), + ("key", "{whispers~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~}", False), + ("key", "{d2hpc3BlcnN+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+}", True), + ("key", "${value$}", False), + ("key", "", False), + ("key", "{value}", False), + ("key", "null", False), + ("key", "!Ref Value", False), + ("key", "{value}", False), + ("key", "/system/path/value", False), + ("thesame", "THESAME", False), + ("label", "WhispersLabel", False), + ("_key", "-key", False), + ("_secret_value_placeholder_", "----SECRET-VALUE-PLACEHOLDER-", False), + ("_secret_value_placeholder_", "----SECRET-VALUE-PLACEHOLDER--", True), + ("SECRET_VALUE_KEY", "whispers", True), + ("whispers", "SECRET_VALUE_PLACEHOLDER", True), + ("secret", "whispers", True), + ], +) +def test_is_static(key, value, expected): + assert is_static(key, value) == expected + + @pytest.mark.parametrize( ("data", "expected"), [ diff --git a/tests/unit/plugins/test_common.py b/tests/unit/plugins/test_common.py new file mode 100644 index 0000000..c11ea5e --- /dev/null +++ b/tests/unit/plugins/test_common.py @@ -0,0 +1,50 @@ +import pytest + +from whispers.plugins.common import Common + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("", None), + (123, None), + ("jdbc:mysql://localhost/authority?userpass=hardcoded0", "hardcoded0"), + ("arn:aws:kms:eu-central-1:123456123456:key/hardcoded", "123456123456"), + ], +) +def test_pairs(text, expected): + plugin = Common() + for pair in plugin.pairs(text): + assert pair.value == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("http", None), + ("jdbc:mysql://localhost/authority?userpass=hardcoded0", "hardcoded0"), + ("jdbc:mysql://localhost/authority?user=&userpass=", None), + ("amqp://root:hardcoded2@localhost.local:5434/topic", "root:hardcoded2"), + ("amqp://root@localhost.local:5434/topic", None), + ("amqp://localhost.local:5434/topic", None), + ], +) +def test_parse_uri(text, expected): + plugin = Common() + for pair in plugin.parse_uri(text): + assert pair.value == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("invalid:", None), + ("arn:aws:kms:eu-central-1", None), + ("arn:aws:kms:eu-central-1:", None), + ("arn:aws:kms:eu-central-1:123456123456:key/hardcoded", "123456123456"), + ], +) +def test_parse_arn(text, expected): + plugin = Common() + for pair in plugin.parse_arn(text): + assert pair.value == expected diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index e74a529..880057a 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -13,4 +13,4 @@ def test_cli(): def test_run(): args = parse_args([fixture_path()]) secrets = list(run(args)) - assert len(secrets) == 294 + assert len(secrets) == 305 diff --git a/whispers/__version__.py b/whispers/__version__.py index f75a07d..fff3d46 100644 --- a/whispers/__version__.py +++ b/whispers/__version__.py @@ -1,4 +1,4 @@ -VERSION = (2, 0, 4) +VERSION = (2, 0, 5) __version__ = ".".join(map(str, VERSION)) diff --git a/whispers/config.yml b/whispers/config.yml index 9fde9f7..c09d024 100644 --- a/whispers/config.yml +++ b/whispers/config.yml @@ -16,5 +16,4 @@ exclude: - ^(true|false|yes|no|1|0)$ - .*_(user|password|token|key|placeholder|name)$ - ^aws_(access_key_id|secret_access_key|session_token)$ - - ^arn:aws:.* - ^((cn?trl|alt|shift|del|ins|esc|tab|f[\d]+) ?[\+_\-\\/] ?)+[\w]+$ diff --git a/whispers/core/args.py b/whispers/core/args.py index 66ebccd..6524586 100644 --- a/whispers/core/args.py +++ b/whispers/core/args.py @@ -10,15 +10,11 @@ def argument_parser() -> ArgumentParser: """CLI argument parser""" args_parser = ArgumentParser("whispers", description=("Identify secrets in static structured text.")) - args_parser.add_argument("-i", "--info", action="store_true", help="show extended help and exit") args_parser.add_argument("-c", "--config", help="config file") args_parser.add_argument( "-C", "--print_config", default=False, action="store_true", help="print default config and exit" ) args_parser.add_argument("-o", "--output", help="output file") - args_parser.add_argument( - "-H", "--human", default=False, action="store_true", help="output in human-readable format" - ) args_parser.add_argument("-e", "--exitcode", default=0, type=int, help="exit code on success") args_parser.add_argument("-f", "--files", help="csv of globs for including files") args_parser.add_argument("-F", "--xfiles", help="regex for excluding files") @@ -28,6 +24,9 @@ def argument_parser() -> ArgumentParser: args_parser.add_argument("-R", "--xrules", help="csv of rule IDs to exclude (see --info)") args_parser.add_argument("-s", "--severity", help="csv of severity levels to report (see --info)") args_parser.add_argument("-S", "--xseverity", help="csv of severity levels to exclude (see --info)") + args_parser.add_argument( + "-H", "--human", default=False, action="store_true", help="output in human-readable format" + ) args_parser.add_argument("-l", "--log", default=False, action="store_true", help="write /tmp/whispers.log") args_parser.add_argument( "-d", @@ -35,9 +34,10 @@ def argument_parser() -> ArgumentParser: action="store_const", const=logging.DEBUG, default=logging.INFO, - help="log debugging information", + help="log debugging information (implies --log)", ) - args_parser.add_argument("-v", "--version", action="version", version=__version__) + args_parser.add_argument("-i", "--info", action="store_true", help="show extended help and exit") + args_parser.add_argument("-v", "--version", action="version", version=__version__, help="show version and exit") args_parser.add_argument("src", nargs="*", help="target file or directory") args_parser.print_help = show_splash(args_parser.print_help) diff --git a/whispers/core/pairs.py b/whispers/core/pairs.py index 900f7b5..958de59 100644 --- a/whispers/core/pairs.py +++ b/whispers/core/pairs.py @@ -3,7 +3,7 @@ from typing import Iterator, Optional from whispers.core.log import global_exception_handler -from whispers.core.utils import REGEX_PRIVKEY_FILE, is_base64_bytes, is_iac, is_path, simple_string, strip_string +from whispers.core.utils import REGEX_PRIVKEY_FILE, is_static, strip_string from whispers.models.appconfig import AppConfig from whispers.models.pair import KeyValuePair from whispers.plugins.config import Config @@ -99,57 +99,6 @@ def filter_static(pair: KeyValuePair) -> Optional[KeyValuePair]: return pair # Static value -def is_static(key: str, value: str) -> bool: - """Check if pair is static""" - if not isinstance(value, str): - return False # Not string - - if not value: - return False # Empty - - if value.lower() == "null": - return False # Empty - - if value.startswith("$") and "$" not in value[2:]: - return False # Variable - - if value.startswith("%") and value.endswith("%"): - return False # Variable - - if value.startswith("${") and value.endswith("}"): - return False # Variable - - if value.startswith("{") and value.endswith("}"): - if len(value) > 50: - if is_base64_bytes(value[1:-1]): - return True # Token - - return False # Variable - - if "{{" in value and "}}" in value: - return False # Variable - - if value.startswith("<") and value.endswith(">"): - return False # Placeholder - - s_key = simple_string(key) - s_value = simple_string(value) - - if s_key == s_value: - return False # Placeholder - - if s_value.endswith(s_key): - return False # Placeholder - - if is_iac(value): - return False # IaC !Ref !Sub ... - - if is_path(value): - return False # System path - - return True # Hardcoded static value - - def load_plugin(file: Path) -> Optional[object]: """ Loads the correct plugin for given file. diff --git a/whispers/core/utils.py b/whispers/core/utils.py index cbb55fc..56f9d95 100644 --- a/whispers/core/utils.py +++ b/whispers/core/utils.py @@ -77,6 +77,60 @@ def similar_strings(a: str, b: str) -> float: return jaro_winkler_similarity(a, b) +def is_static(key: str, value: str) -> bool: + """Check if pair is static""" + if not isinstance(value, str): + return False # Not string + + if not value: + return False # Empty + + if value.lower() == "null": + return False # Empty + + if value.startswith("$") and "$" not in value[2:]: + return False # Variable + + if value.startswith("%") and value.endswith("%"): + return False # Variable + + if value.startswith("${") and value.endswith("}"): + return False # Variable + + if value.startswith("{") and value.endswith("}"): + if len(value) > 50: + if is_base64_bytes(value[1:-1]): + return True # Token + + return False # Variable + + if "{{" in value and "}}" in value: + return False # Variable + + if value.startswith("<") and value.endswith(">"): + return False # Placeholder + + if value.startswith("ENC[AES256_GCM,data:") and value.endswith("]"): + return False # Encrypted SOPS key + + s_key = simple_string(key) + s_value = simple_string(value) + + if s_key == s_value: + return False # Placeholder + + if s_value.endswith(s_key): + return False # Placeholder + + if is_iac(value): + return False # IaC !Ref !Sub ... + + if is_path(value): + return False # System path + + return True # Hardcoded static value + + def is_ascii(data: str) -> bool: """Checks if given data is printable text""" if isinstance(data, bytes): diff --git a/whispers/models/pair.py b/whispers/models/pair.py index 882568e..01e3390 100644 --- a/whispers/models/pair.py +++ b/whispers/models/pair.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from typing import Dict, List @dataclass @@ -7,10 +8,10 @@ class KeyValuePair: key: str value: str - keypath: list = field(default_factory=list) + keypath: List = field(default_factory=list) file: str = "" line: int = 0 - rule: dict = field(default_factory=dict) + rule: Dict = field(default_factory=dict) def __post_init__(self) -> None: if self.keypath == []: diff --git a/whispers/plugins/common.py b/whispers/plugins/common.py new file mode 100644 index 0000000..7c59247 --- /dev/null +++ b/whispers/plugins/common.py @@ -0,0 +1,55 @@ +from itertools import chain +from typing import Iterator, List +from urllib.parse import parse_qsl, urlparse + +from whispers.models.pair import KeyValuePair + + +class Common: + """Checks text for common patterns, such as URI, AWS ARN, etc.""" + + def __init__(self, keypath: List = [], line: int = 0) -> None: + self.keypath = keypath + self.line = line + + def pairs(self, value: str) -> Iterator[KeyValuePair]: + """Check for common patterns in text""" + if not value: + return [] # Empty + + if not isinstance(value, str): + return [] # Not a string + + words = value.split(" ") + uris = map(self.parse_uri, words) + arns = map(self.parse_arn, words) + parsed = chain.from_iterable([*uris, *arns]) + yield from parsed + + def parse_uri(self, text: str) -> Iterator[KeyValuePair]: + """Check if text resembles a Uniform Resource Identifier (URI)""" + if "://" not in text: + return [] # Not URI + + uri = urlparse(text) + + if uri.password: + yield KeyValuePair("uri_creds", f"{uri.username}:{uri.password}", [*self.keypath, text]) + + if uri.query: + for key, value in parse_qsl(uri.query): + yield KeyValuePair(key, value, [*self.keypath, text], line=self.line) + + def parse_arn(self, text: str) -> Iterator[KeyValuePair]: + """Check if text resembles an AWS ARN""" + if not text.startswith("arn:aws:"): + return [] # Not AWS ARN + + arn = text.split(":") + + if len(arn) < 5 or not arn[4]: + return [] # Missing AWS Account ID + + account = str(arn[4]) + + yield KeyValuePair("aws_account", account, [*self.keypath, text], line=self.line) diff --git a/whispers/plugins/plaintext.py b/whispers/plugins/plaintext.py index eeef9bd..706fed0 100644 --- a/whispers/plugins/plaintext.py +++ b/whispers/plugins/plaintext.py @@ -1,9 +1,9 @@ from pathlib import Path -from typing import Iterator, Optional +from typing import Iterator -from whispers.core.utils import is_uri, strip_string +from whispers.core.utils import strip_string from whispers.models.pair import KeyValuePair -from whispers.plugins.uri import Uri +from whispers.plugins.common import Common class Plaintext: @@ -13,28 +13,19 @@ def pairs(self, filepath: Path) -> Iterator[KeyValuePair]: if not line: continue - yield from self.uri_pairs(line, lineno) - yield from self.privatekey_pairs(line, lineno) + yield from self.common_pairs(line, lineno) - @staticmethod - def privatekey_pairs(line: str, lineno: int) -> Optional[Iterator[KeyValuePair]]: - if not (line.startswith("---") and line.endswith("---")): - return None - - if len(line) < 12: - return None - - yield KeyValuePair("key", line, line=lineno) + def common_pairs(self, text: str, lineno: int) -> Iterator[KeyValuePair]: + yield from Common(line=lineno).pairs(text) + yield from self.parse_pk(text) @staticmethod - def uri_pairs(line: str, lineno: int) -> Optional[Iterator[KeyValuePair]]: - if "://" not in line: - return None + def parse_pk(text: str) -> Iterator[KeyValuePair]: + """Check if text resembles a Private Key (PK), only for plaintext files""" + if not (text.startswith("-----") and text.endswith("-----")): + return [] - for value in line.split(): - if not is_uri(value): - continue + if len(text) < 15: + return [] - for pair in Uri().pairs(value): - pair.line = lineno - yield pair + yield KeyValuePair("private_key", text) diff --git a/whispers/plugins/traverse.py b/whispers/plugins/traverse.py index c6accc3..4e1b1d0 100644 --- a/whispers/plugins/traverse.py +++ b/whispers/plugins/traverse.py @@ -1,8 +1,7 @@ from typing import Iterator -from whispers.core.utils import is_uri from whispers.models.pair import KeyValuePair -from whispers.plugins.uri import Uri +from whispers.plugins.common import Common class StructuredDocument: @@ -40,10 +39,7 @@ def traverse(self, code, key=None): if len(item) == 2: yield KeyValuePair(item[0], item[1], list(self.keypath)) - if is_uri(code): - for pair in Uri().pairs(code): - pair.keypath = list(self.keypath) - yield pair + yield from Common(self.keypath).pairs(code) def cloudformation(self, code: dict) -> Iterator[KeyValuePair]: """AWS CloudFormation format""" diff --git a/whispers/plugins/uri.py b/whispers/plugins/uri.py deleted file mode 100644 index 98b2e66..0000000 --- a/whispers/plugins/uri.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Iterator -from urllib.parse import parse_qsl, urlparse - -from whispers.models.pair import KeyValuePair - - -class Uri: - def pairs(self, code: str) -> Iterator[KeyValuePair]: - uri = urlparse(code) - if uri.password: - yield KeyValuePair("uri creds", f"{uri.username}:{uri.password}", [code]) - - if uri.query: - for key, value in parse_qsl(uri.query): - yield KeyValuePair(key, value, [code]) diff --git a/whispers/plugins/xml.py b/whispers/plugins/xml.py index b8bac0b..633305d 100644 --- a/whispers/plugins/xml.py +++ b/whispers/plugins/xml.py @@ -4,9 +4,8 @@ from lxml import etree as ElementTree from whispers.core.log import global_exception_handler -from whispers.core.utils import is_uri from whispers.models.pair import KeyValuePair -from whispers.plugins.uri import Uri +from whispers.plugins.common import Common class Xml: @@ -29,10 +28,7 @@ def _traverse(tree): yield KeyValuePair(key, value, list(self.keypath)) # Format: - if is_uri(value): - for pair in Uri().pairs(value): - pair.keypath = self.keypath + [pair.key] - yield pair + yield from Common(self.keypath).pairs(value) self.keypath.pop() @@ -41,6 +37,7 @@ def _traverse(tree): continue yield KeyValuePair(element.tag, element.text, list(self.keypath)) + yield from Common(self.keypath).pairs(element.text) # Format: key=value if "=" in element.text: @@ -62,6 +59,7 @@ def _traverse(tree): if found_key and found_value: self.keypath.append(found_key) yield KeyValuePair(found_key, found_value, list(self.keypath)) + yield from Common(self.keypath).pairs(found_value) self.keypath.pop() try: @@ -69,5 +67,6 @@ def _traverse(tree): tree = ElementTree.parse(filepath.as_posix(), parser) tree = ElementTree.iterwalk(tree, events=("start", "end")) yield from _traverse(tree) + except Exception: global_exception_handler(filepath.as_posix(), tree) diff --git a/whispers/rules/keys.yml b/whispers/rules/keys.yml index 48b2c3e..16dca27 100644 --- a/whispers/rules/keys.yml +++ b/whispers/rules/keys.yml @@ -32,6 +32,19 @@ ignorecase: False +- id: aws-account + group: keys + description: Values formatted like AWS Account ID + message: AWS Account ID + severity: MINOR + key: + regex: ^aws_account$ + ignorecase: False + value: + regex: ^\d{12}$ + ignorecase: False + + - id: aws-id group: keys description: Values formatted like AWS Access Key ID diff --git a/whispers/rules/passwords.yml b/whispers/rules/passwords.yml index 8129eab..9d73dbf 100644 --- a/whispers/rules/passwords.yml +++ b/whispers/rules/passwords.yml @@ -18,7 +18,7 @@ message: URI Credentials severity: CRITICAL key: - regex: ^uri creds$ + regex: ^uri_creds$ ignorecase: False value: regex: "^\\w+:\\w+$"