Detection (#20)

* Add Mozilla SOPS detection

* Move is_static to utils

* AWS ARN parsing and rule

* Reorder args

* Create common text parser class

* Common operations

* Common string ops

tests/fixtures/arn.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<tests>
+  <compliant>
+      <ok01>aws:arn:</ok01>
+      <ok02>arn:aws:kms:{REGION}:{ACCOUNT}:key/{KEY_ID}</ok02>
+  </compliant>
+  <noncompliant>
+    <arn01>arn:aws:kms:eu-central-1:123456123456:key/hardcoded</arn01>
+    <arn02>arn:aws:kms:ap-southeast-1:123456123456:key/hardcoded</arn02>
+    <arn03>arn:aws:iam::123456123456:oidc-provider/auth-dev.mozilla.auth0.com</arn03>
+  </noncompliant>
+</tests>

tests/fixtures/arn.yml

@@ -0,0 +1,11 @@
+compliant:
+  arn01: "aws:arn:"
+  arn02: arn:aws:kms:{REGION}:{ACCOUNT}:key/{KEY_ID}
+
+
+noncompliant:
+  arn01: arn:aws:kms:eu-central-1:123456123456:key/hardcoded
+  arn02: arn:aws:kms:ap-southeast-1:123456123456:key/hardcoded
+  arn03: arn:aws:iam::123456123456:oidc-provider/auth-dev.mozilla.auth0.com
+  arn_list:
+    - arn:aws:kms:eu-central-1:123456123456:key/hardcoded

tests/fixtures/private-pgp-block.txt

@@ -2,6 +2,7 @@ Compliant:
 
 -----ok
 ---ok---
+-----ok-----
 
 
 Noncompliant:

tests/fixtures/sops.yml

@@ -0,0 +1,7 @@
+# https://github.com/mozilla/sops
+
+compliant:
+  password: ENC[AES256_GCM,data:HARDCODED,iv:1=,aad:No=,tag:k=]
+
+noncompliant:
+  password: hardcoded01

tests/unit/core/test_pairs.py

@@ -6,7 +6,7 @@
 from tests.unit.conftest import FIXTURE_PATH, config_path, fixture_path, forbidden_path, tmp_path
 from whispers.core.args import parse_args
 from whispers.core.config import load_config
-from whispers.core.pairs import filter_included, filter_static, is_static, load_plugin, make_pairs, tag_file
+from whispers.core.pairs import filter_included, filter_static, load_plugin, make_pairs, tag_file
 from whispers.models.pair import KeyValuePair
 from whispers.plugins.config import Config
 from whispers.plugins.dockercfg import Dockercfg
@@ -98,37 +98,6 @@ def test_filter_static(key, value, expected):
     assert filter_static(pair) == expected
 
 
-@pytest.mark.parametrize(
-    ("key", "value", "expected"),
-    [
-        (None, None, False),
-        ("key", "", False),
-        ("key", "$value", False),
-        ("key", "{{value}}", False),
-        ("key", "{value}", False),
-        ("key", "{whispers~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~}", False),
-        ("key", "{d2hpc3BlcnN+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+}", True),
-        ("key", "${value$}", False),
-        ("key", "<value>", False),
-        ("key", "{value}", False),
-        ("key", "null", False),
-        ("key", "!Ref Value", False),
-        ("key", "{value}", False),
-        ("key", "/system/path/value", False),
-        ("thesame", "THESAME", False),
-        ("label", "WhispersLabel", False),
-        ("_key", "-key", False),
-        ("_secret_value_placeholder_", "----SECRET-VALUE-PLACEHOLDER-", False),
-        ("_secret_value_placeholder_", "----SECRET-VALUE-PLACEHOLDER--", True),
-        ("SECRET_VALUE_KEY", "whispers", True),
-        ("whispers", "SECRET_VALUE_PLACEHOLDER", True),
-        ("secret", "whispers", True),
-    ],
-)
-def test_is_static(key, value, expected):
-    assert is_static(key, value) == expected
-
-
 @pytest.mark.parametrize(
     ("filename", "expected"),
     [

tests/unit/core/test_secrets.py

@@ -34,7 +34,7 @@ def test_filter_rule_lineno(rule_fixture):
     ],
 )
 def test_detect_secrets_by_key(src, expected):
-    args = parse_args([fixture_path(src)])
+    args = parse_args(["-S", "MINOR", fixture_path(src)])
     config = load_config(args)
     rules = load_rules(args, config)
     pairs = make_pairs(config, FIXTURE_PATH.joinpath(src))
@@ -54,6 +54,8 @@ def test_detect_secrets_by_key(src, expected):
         ("apikeys.json", "MAJOR", 9),
         ("apikeys.xml", "MAJOR", 9),
         ("apikeys.yml", "MAJOR", 9),
+        ("arn.yml", "MINOR", 4),
+        ("arn.xml", "MINOR", 3),
         ("aws.yml", "BLOCKER", 3),
         ("aws.json", "BLOCKER", 3),
         ("aws.xml", "BLOCKER", 3),
@@ -118,6 +120,7 @@ def test_detect_secrets_by_key(src, expected):
         ("settings01.ini", "CRITICAL", 1),
         ("settings02.ini", "CRITICAL", 1),
         ("severity.yml", "BLOCKER", 1),
+        ("sops.yml", DEFAULT_SEVERITY, 1),
         ("uri.yml", "CRITICAL", 3),
         ("webhooks.yml", "MINOR", 6),
     ],

tests/unit/core/test_utils.py

@@ -16,6 +16,7 @@
     is_iac,
     is_luhn,
     is_path,
+    is_static,
     is_uri,
     list_rule_prop,
     load_regex,
@@ -123,6 +124,37 @@ def test_load_yaml_from_file(configfile, expected, raised):
         assert result == expected
 
 
+@pytest.mark.parametrize(
+    ("key", "value", "expected"),
+    [
+        (None, None, False),
+        ("key", "", False),
+        ("key", "$value", False),
+        ("key", "{{value}}", False),
+        ("key", "{value}", False),
+        ("key", "{whispers~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~}", False),
+        ("key", "{d2hpc3BlcnN+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+}", True),
+        ("key", "${value$}", False),
+        ("key", "<value>", False),
+        ("key", "{value}", False),
+        ("key", "null", False),
+        ("key", "!Ref Value", False),
+        ("key", "{value}", False),
+        ("key", "/system/path/value", False),
+        ("thesame", "THESAME", False),
+        ("label", "WhispersLabel", False),
+        ("_key", "-key", False),
+        ("_secret_value_placeholder_", "----SECRET-VALUE-PLACEHOLDER-", False),
+        ("_secret_value_placeholder_", "----SECRET-VALUE-PLACEHOLDER--", True),
+        ("SECRET_VALUE_KEY", "whispers", True),
+        ("whispers", "SECRET_VALUE_PLACEHOLDER", True),
+        ("secret", "whispers", True),
+    ],
+)
+def test_is_static(key, value, expected):
+    assert is_static(key, value) == expected
+
+
 @pytest.mark.parametrize(
     ("data", "expected"),
     [

tests/unit/plugins/test_common.py

@@ -0,0 +1,50 @@
+import pytest
+
+from whispers.plugins.common import Common
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("", None),
+        (123, None),
+        ("jdbc:mysql://localhost/authority?userpass=hardcoded0", "hardcoded0"),
+        ("arn:aws:kms:eu-central-1:123456123456:key/hardcoded", "123456123456"),
+    ],
+)
+def test_pairs(text, expected):
+    plugin = Common()
+    for pair in plugin.pairs(text):
+        assert pair.value == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("http", None),
+        ("jdbc:mysql://localhost/authority?userpass=hardcoded0", "hardcoded0"),
+        ("jdbc:mysql://localhost/authority?user=&userpass=", None),
+        ("amqp://root:[email protected]:5434/topic", "root:hardcoded2"),
+        ("amqp://[email protected]:5434/topic", None),
+        ("amqp://localhost.local:5434/topic", None),
+    ],
+)
+def test_parse_uri(text, expected):
+    plugin = Common()
+    for pair in plugin.parse_uri(text):
+        assert pair.value == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("invalid:", None),
+        ("arn:aws:kms:eu-central-1", None),
+        ("arn:aws:kms:eu-central-1:", None),
+        ("arn:aws:kms:eu-central-1:123456123456:key/hardcoded", "123456123456"),
+    ],
+)
+def test_parse_arn(text, expected):
+    plugin = Common()
+    for pair in plugin.parse_arn(text):
+        assert pair.value == expected

tests/unit/test_main.py

@@ -13,4 +13,4 @@ def test_cli():
 def test_run():
     args = parse_args([fixture_path()])
     secrets = list(run(args))
-    assert len(secrets) == 294
+    assert len(secrets) == 305

whispers/__version__.py

@@ -1,4 +1,4 @@
-VERSION = (2, 0, 4)
+VERSION = (2, 0, 5)
 
 __version__ = ".".join(map(str, VERSION))
 

whispers/config.yml

@@ -16,5 +16,4 @@ exclude:
     - ^(true|false|yes|no|1|0)$
     - .*_(user|password|token|key|placeholder|name)$
     - ^aws_(access_key_id|secret_access_key|session_token)$
-    - ^arn:aws:.*
     - ^((cn?trl|alt|shift|del|ins|esc|tab|f[\d]+) ?[\+_\-\\/] ?)+[\w]+$

whispers/core/args.py

@@ -10,15 +10,11 @@
 def argument_parser() -> ArgumentParser:
     """CLI argument parser"""
     args_parser = ArgumentParser("whispers", description=("Identify secrets in static structured text."))
-    args_parser.add_argument("-i", "--info", action="store_true", help="show extended help and exit")
     args_parser.add_argument("-c", "--config", help="config file")
     args_parser.add_argument(
         "-C", "--print_config", default=False, action="store_true", help="print default config and exit"
     )
     args_parser.add_argument("-o", "--output", help="output file")
-    args_parser.add_argument(
-        "-H", "--human", default=False, action="store_true", help="output in human-readable format"
-    )
     args_parser.add_argument("-e", "--exitcode", default=0, type=int, help="exit code on success")
     args_parser.add_argument("-f", "--files", help="csv of globs for including files")
     args_parser.add_argument("-F", "--xfiles", help="regex for excluding files")
@@ -28,16 +24,20 @@ def argument_parser() -> ArgumentParser:
     args_parser.add_argument("-R", "--xrules", help="csv of rule IDs to exclude (see --info)")
     args_parser.add_argument("-s", "--severity", help="csv of severity levels to report (see --info)")
     args_parser.add_argument("-S", "--xseverity", help="csv of severity levels to exclude (see --info)")
+    args_parser.add_argument(
+        "-H", "--human", default=False, action="store_true", help="output in human-readable format"
+    )
     args_parser.add_argument("-l", "--log", default=False, action="store_true", help="write /tmp/whispers.log")
     args_parser.add_argument(
         "-d",
         "--debug",
         action="store_const",
         const=logging.DEBUG,
         default=logging.INFO,
-        help="log debugging information",
+        help="log debugging information (implies --log)",
     )
-    args_parser.add_argument("-v", "--version", action="version", version=__version__)
+    args_parser.add_argument("-i", "--info", action="store_true", help="show extended help and exit")
+    args_parser.add_argument("-v", "--version", action="version", version=__version__, help="show version and exit")
     args_parser.add_argument("src", nargs="*", help="target file or directory")
 
     args_parser.print_help = show_splash(args_parser.print_help)

whispers/core/pairs.py

@@ -3,7 +3,7 @@
 from typing import Iterator, Optional
 
 from whispers.core.log import global_exception_handler
-from whispers.core.utils import REGEX_PRIVKEY_FILE, is_base64_bytes, is_iac, is_path, simple_string, strip_string
+from whispers.core.utils import REGEX_PRIVKEY_FILE, is_static, strip_string
 from whispers.models.appconfig import AppConfig
 from whispers.models.pair import KeyValuePair
 from whispers.plugins.config import Config
@@ -99,57 +99,6 @@ def filter_static(pair: KeyValuePair) -> Optional[KeyValuePair]:
     return pair  # Static value
 
 
-def is_static(key: str, value: str) -> bool:
-    """Check if pair is static"""
-    if not isinstance(value, str):
-        return False  # Not string
-
-    if not value:
-        return False  # Empty
-
-    if value.lower() == "null":
-        return False  # Empty
-
-    if value.startswith("$") and "$" not in value[2:]:
-        return False  # Variable
-
-    if value.startswith("%") and value.endswith("%"):
-        return False  # Variable
-
-    if value.startswith("${") and value.endswith("}"):
-        return False  # Variable
-
-    if value.startswith("{") and value.endswith("}"):
-        if len(value) > 50:
-            if is_base64_bytes(value[1:-1]):
-                return True  # Token
-
-        return False  # Variable
-
-    if "{{" in value and "}}" in value:
-        return False  # Variable
-
-    if value.startswith("<") and value.endswith(">"):
-        return False  # Placeholder
-
-    s_key = simple_string(key)
-    s_value = simple_string(value)
-
-    if s_key == s_value:
-        return False  # Placeholder
-
-    if s_value.endswith(s_key):
-        return False  # Placeholder
-
-    if is_iac(value):
-        return False  # IaC !Ref !Sub ...
-
-    if is_path(value):
-        return False  # System path
-
-    return True  # Hardcoded static value
-
-
 def load_plugin(file: Path) -> Optional[object]:
     """
     Loads the correct plugin for given file.

whispers/core/utils.py

@@ -77,6 +77,60 @@ def similar_strings(a: str, b: str) -> float:
     return jaro_winkler_similarity(a, b)
 
 
+def is_static(key: str, value: str) -> bool:
+    """Check if pair is static"""
+    if not isinstance(value, str):
+        return False  # Not string
+
+    if not value:
+        return False  # Empty
+
+    if value.lower() == "null":
+        return False  # Empty
+
+    if value.startswith("$") and "$" not in value[2:]:
+        return False  # Variable
+
+    if value.startswith("%") and value.endswith("%"):
+        return False  # Variable
+
+    if value.startswith("${") and value.endswith("}"):
+        return False  # Variable
+
+    if value.startswith("{") and value.endswith("}"):
+        if len(value) > 50:
+            if is_base64_bytes(value[1:-1]):
+                return True  # Token
+
+        return False  # Variable
+
+    if "{{" in value and "}}" in value:
+        return False  # Variable
+
+    if value.startswith("<") and value.endswith(">"):
+        return False  # Placeholder
+
+    if value.startswith("ENC[AES256_GCM,data:") and value.endswith("]"):
+        return False  # Encrypted SOPS key
+
+    s_key = simple_string(key)
+    s_value = simple_string(value)
+
+    if s_key == s_value:
+        return False  # Placeholder
+
+    if s_value.endswith(s_key):
+        return False  # Placeholder
+
+    if is_iac(value):
+        return False  # IaC !Ref !Sub ...
+
+    if is_path(value):
+        return False  # System path
+
+    return True  # Hardcoded static value
+
+
 def is_ascii(data: str) -> bool:
     """Checks if given data is printable text"""
     if isinstance(data, bytes):