Refactor the operators_test and report test coverage

The operators_test was a badly designed test suite, since it said it was
just testing the operators but in fact it was also testing the process
of parsing them, since it's much easier to specify operations using the
GenericWebhookConfig CRD.

The problem was that the CRD will evolve and the generic-k8s-webhook
should support (and test) both old and new versions of the CRD. The
refactor of the tests makes easier to keep test coverage for all the
versions that are supported.

Apart from that, we're introducing pytest-cov to write a test coverage
report after executing all the unittests.

Makefile

@@ -1,7 +1,7 @@
 SRC_DIR := ./generic_k8s_webhook
 TEST_DIR := ./tests
 SRC_FILES := $(shell find $(SRC_DIR) -type f -name '*.py')
-TEST_FILES := $(shell find $(TEST_DIR) -type f -name '*.py')
+TEST_FILES := $(shell find $(TEST_DIR) -type f -name '*.py' -o -name '*.yaml')
 
 out/install-deps.stamp: pyproject.toml poetry.lock
 	poetry install
@@ -29,7 +29,8 @@ format: build
 
 .PHONY: unittests
 unittests: build
-	poetry run pytest tests --timeout 15
+	poetry run pytest tests
+	poetry run coverage html
 
 .PHONY: check-pyproject
 check-pyproject:

docs/contributor-guide.md

@@ -65,6 +65,12 @@ We use [pytest](https://docs.pytest.org/en/7.3.x/) to test the functionality of
 make unittests
 ```
 
+The previous command will generate a test coverage report. You can open it on your browser by typing:
+
+```bash
+open htmlcov/index.html
+```
+
 ### Docker build
 
 The last phase of our testing suite is building the docker container that has our app installed in it.

pyproject.toml

@@ -22,6 +22,7 @@ pytest-timeout = "^2.1.0"
 isort = "^5.12.0"
 black = "^23.3.0"
 pylint = "^2.17.4"
+pytest-cov = "^5.0.0"
 
 [tool.isort]
 line_length = 120
@@ -40,6 +41,9 @@ too-few-public-methods, \
 fixme, \
 """
 
+[tool.pytest.ini_options]
+addopts = "-v --timeout=15 --cov=generic_k8s_webhook"
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

tests/conditions_test.py

@@ -0,0 +1,52 @@
+import os
+
+import pytest
+import yaml
+
+from generic_k8s_webhook.config_parser.entrypoint import GenericWebhookConfigManifest
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+CONDITIONS_YAML = os.path.join(SCRIPT_DIR, "conditions_test.yaml")
+
+
+def _expand_schemas(schemas_subsets: dict[str, list[str]], list_schemas: list[str]) -> list[str]:
+    final_schemas = set()
+    for schema in list_schemas:
+        final_schemas.add(schema)
+        for schemas_superset in schemas_subsets.get(schema, []):
+            final_schemas.add(schemas_superset)
+    return sorted(list(final_schemas))
+
+
+def _parse_tests() -> list[tuple]:
+    with open(CONDITIONS_YAML, "r") as f:
+        raw_tests = yaml.safe_load(f)
+
+    parsed_tests = []
+    for test_suite in raw_tests["test_suites"]:
+        for test in test_suite["tests"]:
+            for schema in _expand_schemas(raw_tests["schemas_subsets"], test["schemas"]):
+                for i, case in enumerate(test["cases"]):
+                    parsed_tests.append(
+                        (
+                            f"{test_suite['name']}_{i}",  # name
+                            schema,  # schema
+                            case["condition"],  # condition
+                            case.get("context", [{}]),  # context
+                            case["expected_result"],  # expected_result
+                        )
+                    )
+    return parsed_tests
+
+
+@pytest.mark.parametrize(("name", "schema", "condition", "context", "expected_result"), _parse_tests())
+def test_all(name, schema, condition, context, expected_result):
+    raw_config = {
+        "apiVersion": f"generic-webhook/{schema}",
+        "kind": "GenericWebhookConfig",
+        "webhooks": [{"name": "test-webhook", "path": "test-path", "actions": [{"condition": condition}]}],
+    }
+    gwcm = GenericWebhookConfigManifest(raw_config)
+    action = gwcm.list_webhook_config[0].list_actions[0]
+    result = action.condition.get_value(context)
+    assert result == expected_result

tests/conditions_test.yaml

@@ -0,0 +1,193 @@
+schemas_subsets:
+  # v1alpha1 is a subset of the features from v1beta1
+  # For this reason, any test that uses v1alpha1 will also be executed for v1beta1
+  v1alpha1:
+    - v1beta1
+
+# Each suite stresses a specific operator or set of operators used to specify conditions
+# Each test specifies the schema used to parse the condition. Notice that the same test
+# will be executed by all the schemas that are a superset of the one specified in the test.
+# A test is composed by different cases. Each case defines the actual condition to be tested,
+# some context (optional) and the expected result.
+test_suites:
+  - name: AND
+    tests:
+      - schemas: [v1alpha1]
+        cases:
+          - condition:
+              and:
+                - const: true
+                - const: true
+            expected_result: true
+          - condition:
+              and:
+                - const: true
+                - const: false
+            expected_result: false
+          - condition:
+              and:
+                - const: false
+            expected_result: false
+          - condition:
+              and: []
+            expected_result: true
+  - name: OR
+    tests:
+      - schemas: [v1alpha1]
+        cases:
+          - condition:
+              or:
+                - const: false
+                - const: false
+            expected_result: false
+          - condition:
+              or:
+                - const: true
+                - const: false
+            expected_result: true
+          - condition:
+              or:
+                - const: true
+            expected_result: true
+  - name: NOT
+    tests:
+      - schemas: [v1alpha1]
+        cases:
+          - condition:
+              not:
+                const: true
+            expected_result: false
+          - condition:
+              not:
+                const: false
+            expected_result: true
+  - name: EQUAL
+    tests:
+      - schemas: [v1alpha1]
+        cases:
+          - condition:
+              equal:
+                - const: 1
+            expected_result: true
+          - condition:
+              equal:
+                - const: 1
+                - const: 2
+            expected_result: false
+          - condition:
+              equal:
+                - const: 2
+                - const: 2
+            expected_result: true
+  - name: SUM
+    tests:
+      - schemas: [v1alpha1]
+        cases:
+          - condition:
+              sum:
+                - const: 1
+                - const: 2
+                - const: 3
+            expected_result: 6
+          - condition:
+              sum:
+                - const: 2
+            expected_result: 2
+  - name: GET_VALUE
+    tests:
+      - schemas: [v1alpha1]
+        cases:
+          # Retrieve value from last context
+          - condition:
+              getValue: .name
+            context:
+              - metadata:
+                  name: foo
+                  spec: {}
+              - name: bar
+            expected_result: bar
+          # Retrieve value from first context
+          - condition:
+              getValue: $.metadata.name
+            context:
+              - metadata:
+                  name: foo
+                  spec: {}
+              - name: bar
+            expected_result: foo
+  - name: FOR_EACH
+    tests:
+      - schemas: [v1alpha1]
+        cases:
+          # Iterate over a constant list of elements and sum 10 to each
+          - condition:
+              forEach:
+                elements:
+                  const: [1, 2]
+                op:
+                  sum:
+                    - const: 10
+                    - getValue: "."
+            expected_result: [11, 12]
+          # Iterate over a list defined in the yaml file and sum 1 to each
+          - condition:
+              forEach:
+                elements:
+                  getValue: .containers
+                op:
+                  sum:
+                    - const: 1
+                    - getValue: .maxCPU
+            context:
+              - containers:
+                  - maxCPU: 1
+                  - maxCPU: 2
+            expected_result: [2, 3]
+  - name: CONTAIN
+    tests:
+      - schemas: [v1alpha1]
+        cases:
+          - condition:
+              contain:
+                elements:
+                  getValue: .containers
+                value:
+                  const: { maxCPU: 2 }
+            context:
+              - containers:
+                  - maxCPU: 1
+                  - maxCPU: 2
+            expected_result: true
+          - condition:
+              contain:
+                elements:
+                  getValue: .containers
+                value:
+                  const: { maxCPU: 4 }
+            context:
+              - containers:
+                  - maxCPU: 1
+                  - maxCPU: 2
+            expected_result: false
+  - name: RAW_STR_EXPR
+    tests:
+      - schemas: [v1beta1]
+        cases:
+          - condition: "2 * (3 + 4 / 2) - 1"
+            expected_result: 9
+          - condition: "2*(3+4/2)-1"
+            expected_result: 9
+          - condition: "8/4/2"
+            expected_result: 1
+          - condition: "1 == 1 && 1 != 0 && 0 <= 0 && 0 < 1 && 1 > 0 && 1 >= 1 && true"
+            expected_result: true
+          - condition: "1 != 1 || 1 == 0 || 0 < 0 || 0 >= 1 || 1 <= 0 || 1 < 1 || false"
+            expected_result: false
+          - condition: '"foo" == "foo" && "foo" != "bar"'
+            expected_result: true
+          - condition: ".containers.0.maxCPU + 1  == .containers.1.maxCPU"
+            context:
+              - containers:
+                  - maxCPU: 1
+                  - maxCPU: 2
+            expected_result: true