SecretsManager: list_secrets() now properly splits words when filteri…

…ng all values (getmoto#8469)

moto/secretsmanager/list_secrets/filters.py

@@ -1,3 +1,4 @@
+import re
 from typing import TYPE_CHECKING, List
 
 if TYPE_CHECKING:
@@ -9,7 +10,14 @@ def name_filter(secret: "FakeSecret", names: List[str]) -> bool:
 
 
 def description_filter(secret: "FakeSecret", descriptions: List[str]) -> bool:
-    return _matcher(descriptions, [secret.description], match_prefix=False)  # type: ignore
+    if not secret.description:
+        return False
+    # The documentation states that this search uses `Prefix match`
+    # But actual testing determines that it uses the same approach to the `all_filter`:
+    # 'Breaks the filter value string into words and then searches all attributes for matches.'
+    return _matcher(
+        descriptions, [secret.description], match_prefix=False, case_sensitive=False
+    )
 
 
 def tag_key(secret: "FakeSecret", tag_keys: List[str]) -> bool:
@@ -25,40 +33,66 @@ def tag_value(secret: "FakeSecret", tag_values: List[str]) -> bool:
 
 
 def filter_all(secret: "FakeSecret", values: List[str]) -> bool:
-    attributes = [secret.name, secret.description]
+    attributes = [secret.name]
+    if secret.description:
+        attributes.append(secret.description)
     if secret.tags:
         attributes += [tag["Key"] for tag in secret.tags] + [
             tag["Value"] for tag in secret.tags
         ]
 
-    return _matcher(values, attributes)  # type: ignore
+    return _matcher(values, attributes, match_prefix=False, case_sensitive=False)
 
 
 def _matcher(
-    patterns: List[str], strings: List[str], match_prefix: bool = True
+    patterns: List[str],
+    strings: List[str],
+    match_prefix: bool = True,
+    case_sensitive: bool = True,
 ) -> bool:
     for pattern in [p for p in patterns if p.startswith("!")]:
         for string in strings:
-            if not _match_pattern(pattern[1:], string, match_prefix):
+            if not _match_pattern(
+                pattern[1:], string, match_prefix, case_sensitive=case_sensitive
+            ):
                 return True
 
     for pattern in [p for p in patterns if not p.startswith("!")]:
         for string in strings:
-            if _match_pattern(pattern, string, match_prefix):
+            if _match_pattern(
+                pattern, string, match_prefix, case_sensitive=case_sensitive
+            ):
                 return True
     return False
 
 
-def _match_pattern(pattern: str, value: str, match_prefix: bool = True) -> bool:
+def _match_pattern(
+    pattern: str, value: str, match_prefix: bool = True, case_sensitive: bool = True
+) -> bool:
     if match_prefix:
-        return value.startswith(pattern)
+        if not case_sensitive:
+            return value.lower().startswith(pattern.lower())
+        else:
+            return value.startswith(pattern)
     else:
-        pattern_words = pattern.split(" ")
-        value_words = value.split(" ")
+        pattern_words = split_words(pattern)
+        value_words = split_words(value)
+        if not case_sensitive:
+            pattern_words = [p.lower() for p in pattern_words]
+            value_words = [v.lower() for v in value_words]
         for pattern_word in pattern_words:
             # all words in value must start with pattern_word
             if not any(
                 value_word.startswith(pattern_word) for value_word in value_words
             ):
                 return False
     return True
+
+
+def split_words(s: str) -> List[str]:
+    """
+    Split a string into words. Words are recognized by upper case letters, i.e.:
+    test   -> [test]
+    MyTest -> [My, Test]
+    """
+    return [x.strip() for x in re.split(r"([^a-z][a-z]+)", s) if x]

tests/test_secretsmanager/test_list_secrets.py

@@ -1,11 +1,14 @@
 from datetime import datetime
+from uuid import uuid4
 
 import boto3
 import pytest
 from botocore.exceptions import ClientError
 from dateutil.tz import tzlocal
 
 from moto import mock_aws
+from moto.secretsmanager.list_secrets.filters import split_words
+from tests import aws_verified
 
 
 def boto_client():
@@ -104,30 +107,111 @@ def test_with_description_filter():
     assert secret_names == ["foo"]
 
 
-@mock_aws
+@aws_verified
+# Verified, but not marked because it's flaky - AWS can take up to 5 minutes before secrets are listed
 def test_with_all_filter():
     # The 'all' filter will match a secret that contains ANY field with
     # the criteria. In other words an implicit OR.
+    unique_name = str(uuid4())[0:6]
+
+    first_secret = f"SecretOne{unique_name}"
+    second_secret = f"SecretTwo{unique_name}"
+    third_secret = f"Thirdsecret{unique_name}"
+    foo_tag_key = f"SecretFTag{unique_name}"
+    foo_tag_val = f"SecretFValue{unique_name}"
+    no_match = f"none{unique_name}"
 
     conn = boto_client()
 
-    conn.create_secret(Name="foo", SecretString="secret")
-    conn.create_secret(Name="bar", SecretString="secret", Description="foo")
-    conn.create_secret(
-        Name="baz", SecretString="secret", Tags=[{"Key": "foo", "Value": "1"}]
-    )
+    conn.create_secret(Name=first_secret, SecretString="s")
+    conn.create_secret(Name=second_secret, SecretString="s", Description="DescTwo")
+    conn.create_secret(Name=third_secret, SecretString="s")
     conn.create_secret(
-        Name="qux", SecretString="secret", Tags=[{"Key": "1", "Value": "foo"}]
+        Name=foo_tag_key, SecretString="s", Tags=[{"Key": "1", "Value": "foo"}]
     )
     conn.create_secret(
-        Name="multi", SecretString="secret", Tags=[{"Key": "foo", "Value": "foo"}]
+        Name=foo_tag_val, SecretString="s", Tags=[{"Key": "foo", "Value": "v"}]
     )
-    conn.create_secret(Name="none", SecretString="secret")
+    conn.create_secret(Name=no_match, SecretString="s")
 
-    secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["foo"]}])
+    try:
+        # Full Match
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["SecretOne"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert secret_names == [first_secret]
 
-    secret_names = list(map(lambda s: s["Name"], secrets["SecretList"]))
-    assert sorted(secret_names) == ["bar", "baz", "foo", "multi", "qux"]
+        # Search in tags
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["foo"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert secret_names == [foo_tag_key, foo_tag_val]
+
+        # StartsWith - full word
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["Secret"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert secret_names == [first_secret, second_secret, foo_tag_key, foo_tag_val]
+
+        # Partial Match - full word - lowercase
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["secret"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert secret_names == [first_secret, second_secret, foo_tag_key, foo_tag_val]
+
+        # Partial Match - partial word
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["ret"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert not secret_names
+
+        # Partial Match - partial word
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["sec"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert secret_names == [first_secret, second_secret, foo_tag_key, foo_tag_val]
+
+        # Unknown Match
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["SomeSecret"]}])[
+            "SecretList"
+        ]
+        assert not secrets
+
+        # Single value
+        # Only matches description that contains a d
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["d"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert secret_names == [second_secret]
+
+        # Multiple values
+        # Only matches description that contains a d and t (DescTwo)
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["d t"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert secret_names == [second_secret]
+
+        # Name parts that start with a t (DescTwo)
+        secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["t"]}])[
+            "SecretList"
+        ]
+        secret_names = [s["Name"] for s in secrets]
+        assert secret_names == [second_secret, third_secret, foo_tag_key]
+    finally:
+        conn.delete_secret(SecretId=first_secret, ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId=second_secret, ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId=third_secret, ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId=foo_tag_key, ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId=foo_tag_val, ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId=no_match, ForceDeleteWithoutRecovery=True)
 
 
 @mock_aws
@@ -171,7 +255,8 @@ def test_with_invalid_filter_key():
     )
 
 
-@mock_aws
+@aws_verified
+# Verified, but not marked because it's flaky - AWS can take up to 5 minutes before secrets are listed
 def test_with_duplicate_filter_keys():
     # Multiple filters with the same key combine with an implicit AND operator
 
@@ -182,15 +267,21 @@ def test_with_duplicate_filter_keys():
     conn.create_secret(Name="baz", SecretString="secret", Description="two")
     conn.create_secret(Name="qux", SecretString="secret", Description="unrelated")
 
-    secrets = conn.list_secrets(
-        Filters=[
-            {"Key": "description", "Values": ["one"]},
-            {"Key": "description", "Values": ["two"]},
-        ]
-    )
+    try:
+        secrets = conn.list_secrets(
+            Filters=[
+                {"Key": "description", "Values": ["one"]},
+                {"Key": "description", "Values": ["two"]},
+            ]
+        )
 
-    secret_names = list(map(lambda s: s["Name"], secrets["SecretList"]))
-    assert secret_names == ["foo"]
+        secret_names = list(map(lambda s: s["Name"], secrets["SecretList"]))
+        assert secret_names == ["foo"]
+    finally:
+        conn.delete_secret(SecretId="foo", ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId="bar", ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId="baz", ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId="qux", ForceDeleteWithoutRecovery=True)
 
 
 @mock_aws
@@ -237,7 +328,8 @@ def test_with_filter_with_multiple_values():
     assert secret_names == ["foo", "bar"]
 
 
-@mock_aws
+@aws_verified
+# Verified, but not marked because it's flaky - AWS can take up to 5 minutes before secrets are listed
 def test_with_filter_with_value_with_multiple_words():
     conn = boto_client()
 
@@ -247,10 +339,29 @@ def test_with_filter_with_value_with_multiple_words():
     conn.create_secret(Name="qux", SecretString="secret", Description="two")
     conn.create_secret(Name="none", SecretString="secret", Description="unrelated")
 
-    secrets = conn.list_secrets(Filters=[{"Key": "description", "Values": ["one two"]}])
-
-    secret_names = list(map(lambda s: s["Name"], secrets["SecretList"]))
-    assert secret_names == ["foo", "bar"]
+    try:
+        # All values that contain one and two
+        secrets = conn.list_secrets(
+            Filters=[{"Key": "description", "Values": ["one two"]}]
+        )
+        secret_names = list(map(lambda s: s["Name"], secrets["SecretList"]))
+        assert secret_names == ["foo", "bar"]
+
+        # All values that start with o and t
+        secrets = conn.list_secrets(Filters=[{"Key": "description", "Values": ["o t"]}])
+        secret_names = list(map(lambda s: s["Name"], secrets["SecretList"]))
+        assert secret_names == ["foo", "bar"]
+
+        # All values that contain t
+        secrets = conn.list_secrets(Filters=[{"Key": "description", "Values": ["t"]}])
+        secret_names = list(map(lambda s: s["Name"], secrets["SecretList"]))
+        assert secret_names == ["foo", "bar", "qux"]
+    finally:
+        conn.delete_secret(SecretId="foo", ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId="bar", ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId="baz", ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId="qux", ForceDeleteWithoutRecovery=True)
+        conn.delete_secret(SecretId="none", ForceDeleteWithoutRecovery=True)
 
 
 @mock_aws
@@ -318,3 +429,19 @@ def test_with_include_planned_deleted_secrets():
     assert secrets["SecretList"][0]["ARN"] is not None
     assert secrets["SecretList"][0]["Name"] == "foo"
     assert secrets["SecretList"][0]["SecretVersionsToStages"] is not None
+
+
+@pytest.mark.parametrize(
+    "input,output",
+    [
+        ("test", ["test"]),
+        ("my test", ["my", "test"]),
+        ("Mytest", ["Mytest"]),
+        ("MyTest", ["My", "Test"]),
+        ("MyTestPhrase", ["My", "Test", "Phrase"]),
+        ("myTest", ["my", "Test"]),
+        ("my test", ["my", "test"]),
+    ],
+)
+def test_word_splitter(input, output):
+    assert split_words(input) == output