add enums for test type and candence, additional config checks, remov…

…e extra assert True, and log message for custom test failures

tests/custom_test.py

@@ -12,13 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import runpy
 import unittest
 from typing import Optional
 
 from tests.data import CustomTestConfig
 
 
+_LOGGER = logging.getLogger(__name__)
+
+
 class CustomTestCase(unittest.TestCase):
     """
     CustomTestCase class. All custom test classes written should inherit from this
@@ -78,5 +82,5 @@ def test_custom_class(self, config: Optional[CustomTestConfig] = None):
             raise Exception(output[-1])
 
         for out in output.failures:
+            _LOGGER.error(out[-1])
             assert False
-        assert True

tests/data.py

@@ -13,17 +13,26 @@
 # limitations under the License.
 
 from dataclasses import dataclass
-from typing import List, Union
+from enum import Enum
 
 
-# TODO: add enums
-# test type as decorators?
+# TODO: maybe test type as decorators?
+class TestType(Enum):
+    SANITY = "sanity"
+    REGRESSION = "regression"
+    SMOKE = "smoke"
+
+
+class Cadence(Enum):
+    COMMIT = "commit"
+    WEEKLY = "weekly"
+    NIGHTLY = "nightly"
 
 
 @dataclass
 class TestConfig:
-    test_type: str  # sanity, regression, smoke
-    cadence: Union[str, List]  # weekly, nightly, commit
+    test_type: TestType
+    cadence: Cadence
 
 
 @dataclass

tests/testing_utils.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import dataclasses
+import enum
 import logging
 import os
 import unittest
@@ -60,6 +61,12 @@ def _validate_test_config(config: dict):
     for f in dataclasses.fields(TestConfig):
         if f.name not in config:
             return False
+        config_value = config.get(f.name)
+        if issubclass(f.type, enum.Enum):
+            try:
+                f.type(config_value)
+            except ValueError:
+                raise False
     return True
 
 
@@ -88,10 +95,15 @@ def parse_params(
             if type == "custom":
                 config = CustomTestConfig(**config)
             else:
-                _validate_test_config(config)
+                if not _validate_test_config(config):
+                    raise ValueError(
+                        "The config provided does not comply with the expected "
+                        "structure. See tests.data.TestConfig for the expected "
+                        "fields."
+                    )
             config_dicts.append(config)
         else:
             logging.info(
-                f"Skipping testing model: {file} " f"for cadence: {config['cadence']}"
+                f"Skipping testing model: {file} for cadence: {config['cadence']}"
             )
     return config_dicts