continue magic probe modle

Signed-off-by: Fabian Klemm <[email protected]>

everest-testing/src/everest/testing/core_utils/_configuration/everest_environment_setup.py

@@ -25,6 +25,7 @@
     ProbeModuleConfigurationStrategy
 from .libocpp_configuration_helper import \
     LibOCPP201ConfigurationHelper, LibOCPP16ConfigurationHelper
+from .._magic_probe_module.magic_probe_module_configurator import MagicProbeModuleConfigurator
 
 
 @dataclass
@@ -68,6 +69,8 @@ class EverestEnvironmentCoreConfiguration:
 class EverestEnvironmentProbeModuleConfiguration:
     connections: Dict[str, List[Requirement]] = field(default_factory=dict)
     module_id: str = "probe"
+    magic_probe_module_strict_mode: bool = False,
+    magic_probe_module_configurator: Optional[MagicProbeModuleConfigurator] = None
 
 
 class EverestTestEnvironmentSetup:
@@ -225,6 +228,8 @@ def _create_everest_configuration_strategies(self, temporary_paths: _EverestEnvi
             configuration_strategies.append(
                 ProbeModuleConfigurationStrategy(connections=self._probe_config.connections,
                                                  module_id=self._probe_config.module_id))
+            if self._probe_config.magic_probe_module_configurator:
+                configuration_strategies.append(self._probe_config.magic_probe_module_configurator.get_configuration_adjustment_strategy())
 
         if self._evse_security_config:
             configuration_strategies.append(

everest-testing/src/everest/testing/core_utils/_magic_probe_module/README.md

@@ -0,0 +1,136 @@
+# The Magic Probe Module
+
+## Motivation
+
+The "pure" Probe Module comes with the issue of high configuration effort. To "quickly" test another's module interface,
+a lot of configuration is required. Furthermore, if a probe module's commands are called that are not explicitly implemented,
+the process hangs without further notification.
+
+The idea behind the "magic" probe module is similar to the distinction of the "Mock" and "MagicMock" classes in
+Python unittests: The latter "out of the box" provides all magic methods (for comparisons etc.) and requires less configuration.
+
+The Magic Probe Module should configure itself automatically as much as possible and mitigate the drawbacks of the default
+one.
+
+
+## Core Concepts
+
+The Magic Probe Module:
+- Gets activated by the marker `@magic_probe_module` and this then available via the `magic_probe_module` fixture.
+- Is automatically configured to fulfill _any_ requirement that is not fulfilled in the provided Everest configuration.
+- Also detects existing requirement fulfillments in the provided Everest configuration (in case certain implementation ids shall be useed)
+- Automatically implements _any_ command and wraps it into mock calls. The `implement_command` method of the Magic Probe Module
+allows to set the return value or side effect of each command. This can be done even after module startup. Per default,
+it provides a auto-generated value. Can be set (via the strict parameter of the `@magic_probe_module` to _strict_ mode to
+raise an Exception per default instead)
+- Possesses a value generator that is capable of generating any EVerest type (as far as specified in the Json Schema; works most of the timel™)
+
+## Requirements
+
+The magic probe module currently requires packages:
+- pydantic >= 2
+- rstr 
+
+## Usage Examples
+
+Given the config:
+
+```yaml
+active_modules:
+  ocpp:
+    module: OCPP201
+    config_module:
+      ChargePointConfigPath: config.json
+      EnableExternalWebsocketControl: true
+    connections: {}
+x-module-layout: {}
+
+
+```
+
+the following example (assuming correct imports) works:
+
+```python
+
+@pytest.mark.asyncio
+@pytest.mark.ocpp_version("ocpp2.0.1")
+@pytest.mark.everest_core_config("everest-config-ocpp201-magic-probe-module.yaml")
+@pytest.mark.magic_probe_module()
+async def test_against_occp(central_system: CentralSystem,
+                            test_controller,
+                            magic_probe_module,
+                            test_utility: TestUtility):
+    test_controller.start()
+    magic_probe_module.start()
+    await magic_probe_module.wait_to_be_ready()
+    await central_system.wait_for_chargepoint()
+
+    # get all implementations for a certain interface
+    evse_managers = [impl for impl, intf in magic_probe_module.get_interface_implementations().items() if
+                     intf.interface == "evse_manager"] 
+
+    for evse_manager_implementation in evse_managers:
+        magic_probe_module.publish_variable(evse_manager_implementation, "iso15118_certificate_request",
+                                            {"exiRequest": "bla",
+                                             "iso15118SchemaVersion": "mock_iso15118_schema_version",
+                                             "certificateAction": "ba"})
+        await asyncio.sleep(1)
+        # The OCPP module should have called the "enable" endpoint of the evse_manager, this command is automatically implemented
+        magic_probe_module.implementation_mocks[evse_manager_implementation].enable.assert_called_with(
+            {"connector_id": ANY})
+
+```
+
+Note that under the hood the Magic Probe Module will create the following configuration that EVerest is started with:
+```yaml
+active_modules:
+  ocpp:
+    config_module:
+      CertsPath: /tmp/sharedtmp/pytest/test_against_occp0/certs
+      ChargePointConfigPath: /tmp/sharedtmp/pytest/test_against_occp0/ocpp_config/config.json
+      CoreDatabasePath: /tmp/sharedtmp/pytest/test_against_occp0/ocpp_config
+      DeviceModelDatabasePath: /tmp/sharedtmp/pytest/test_against_occp0/ocpp_config/device_model_storage.db
+      EnableExternalWebsocketControl: true
+      MessageLogPath: /tmp/sharedtmp/pytest/test_against_occp0/ocpp_config/logs
+    connections:
+      evse_manager:
+      - implementation_id: ProbeModuleConnectorA
+        module_id: probe
+      kvs:
+      - implementation_id: kvs
+        module_id: probe
+      security:
+      - implementation_id: evse_security
+        module_id: probe
+      system:
+      - implementation_id: system
+        module_id: probe
+    module: OCPP201
+  probe:
+    connections:
+      auth_token_provider:
+      - implementation_id: auth_provider
+        module_id: ocpp
+      auth_token_validator:
+      - implementation_id: auth_validator
+        module_id: ocpp
+      empty:
+      - implementation_id: main
+        module_id: ocpp
+      ocpp_data_transfer:
+      - implementation_id: data_transfer
+        module_id: ocpp
+    module: ProbeModule
+settings:
+  controller_port: 0
+  mqtt_everest_prefix: everest_5b6796904fe04146b0b79dd07de69c65
+  mqtt_external_prefix: external_5b6796904fe04146b0b79dd07de69c65
+  telemetry_prefix: telemetry_5b6796904fe04146b0b79dd07de69c65
+x-module-layout: {}
+
+```
+
+## Todos
+
+- Issue with invalid values (e.g. Connector ids must be sequential)
+- Downgrade to Pydantic 1

everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module.py

@@ -8,6 +8,7 @@
 from .types.everest_command import EverestCommand
 from .types.everest_interface import EverestInterface
 from .types.everest_type import EverestType
+from .types.json_schema_models import JsonSchema
 from .value_generator import ValueGenerator
 
 class MagicProbeModule(ProbeModule):
@@ -36,6 +37,12 @@ def __init__(self,
         self._implementation_mocks: dict[str, Mock] = {}
         self._init_commands()
 
+    async def wait_to_be_ready(self, timeout=3):
+        await super().wait_to_be_ready(timeout=timeout)
+        for interface_implementation in self._interface_implementations:
+            self.publish_variable(interface_implementation,"ready", True)
+
+
     def implement_command(self, implementation_id: str, command_name: str, handler: Callable[[dict], Any]):
         getattr(self._implementation_mocks[implementation_id], command_name).side_effect = handler
 
@@ -95,10 +102,16 @@ def _handler(*args, **kwargs):
                 return res
             elif command.result:
                 if self._strict_mode:
-                    raise NotImplementedError(f"MagicProbeModule command {implementation_id} / {command} not implemented")
+                    error = f"MagicProbeModule command {implementation_id} / {command} not implemented - aborting test"
+                    logging.error(error)
+                    raise NotImplementedError(error)
                 logging.warning(f"MagicProbeModule command {implementation_id} / {command} not implemented - returning auto-generated value!")
                 return self._value_generator.generate(command.result)
             else:
                 return None
 
         return _handler
+
+
+    def get_value(self, what: EverestType | JsonSchema | str):
+        self._value_generator.generate(what)

everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module_configurator.py

@@ -1,14 +1,12 @@
 import logging
 from dataclasses import dataclass
-from typing import Dict
 
+from everest.testing.core_utils import EverestConfigAdjustmentStrategy
 from everest.testing.core_utils.common import Requirement
-
 from .magic_probe_module_config_strategy import MagicProbeModuleConfigurationStrategy
 from .types.everest_config_schema import EverestConfigSchema
 from .types.everest_interface import EverestInterface
 from .types.everest_module_manifest_schema import EverestModuleManifestSchema
-from everest.testing.core_utils import EverestConfigAdjustmentStrategy
 
 
 @dataclass
@@ -168,3 +166,5 @@ def _determine_probe_module_requirements(self) -> dict[str, dict[str, list[tuple
                          self._interfaces_by_name[provides.interface])
                     )
         return probe_module_requirements
+
+

everest-testing/src/everest/testing/core_utils/everest_core.py

@@ -193,13 +193,17 @@ def start(self, standalone_module: Optional[Union[str, List[str]]] = None, test_
 
         expected_status = 'ALL_MODULES_STARTED' if standalone_module == None else 'WAITING_FOR_STANDALONE_MODULES'
 
-        status = self.status_listener.wait_for_status(STARTUP_TIMEOUT, [expected_status])
-        if status == None or len(status) == 0:
-            raise TimeoutError("Timeout while waiting for EVerest to start")
-
-        logging.info("EVerest has started")
-        if expected_status == 'ALL_MODULES_STARTED':
+        if standalone_module and set(standalone_module) - {"probe"}:
+            time.sleep(25)
             self.all_modules_started_event.set()
+        else:
+            status = self.status_listener.wait_for_status(STARTUP_TIMEOUT, [expected_status])
+            if status == None or len(status) == 0:
+                raise TimeoutError("Timeout while waiting for EVerest to start")
+
+            logging.info("EVerest has started")
+            if expected_status == 'ALL_MODULES_STARTED':
+                self.all_modules_started_event.set()
 
     def read_everest_log(self):
         while self.process.poll() == None:

everest-testing/src/everest/testing/core_utils/fixtures.py

@@ -14,17 +14,33 @@
 from everest.testing.core_utils.controller.everest_test_controller import EverestTestController
 from everest.testing.core_utils.everest_core import EverestCore
 
+from ._magic_probe_module.magic_probe_module_configurator import MagicProbeModuleConfigurator
+from ._magic_probe_module.parser.everest_interface_parser import EverestInterfaceParser
+from ._magic_probe_module.types.everest_config_schema import EverestConfigSchema as _EverestConfigSchema
+from ._magic_probe_module.types.everest_module_manifest_schema import EverestModuleManifestSchema
+import yaml
+
+def _build_magic_probe_module_configurator(core_config: EverestEnvironmentCoreConfiguration) -> MagicProbeModuleConfigurator:
+    everest_config = _EverestConfigSchema(**yaml.safe_load(core_config.template_everest_config_path.read_text()))
+
+    interfaces_directory = core_config.everest_core_path / "share" / "everest" / "interfaces"
+    modules_dir = core_config.everest_core_path / "libexec" / "everest" / "modules"
+
+    everest_interfaces = EverestInterfaceParser().parse([interfaces_directory])
+
+    everest_manifests = {}
+    for f in modules_dir.glob("*"):
+        if (f / "manifest.yaml").exists():
+            everest_manifests[f.name] = EverestModuleManifestSchema(
+                **yaml.safe_load((f / "manifest.yaml").read_text()))
+
+    configurator = MagicProbeModuleConfigurator(
+        everest_config=everest_config,
+        interfaces=everest_interfaces,
+        manifests=everest_manifests
+    )
 
-@pytest.fixture
-def probe_module_config(request) -> Optional[EverestEnvironmentProbeModuleConfiguration]:
-    marker = request.node.get_closest_marker("probe_module")
-    if marker:
-        return EverestEnvironmentProbeModuleConfiguration(
-            **marker.kwargs
-        )
-
-    return None
-
+    return configurator
 
 @pytest.fixture
 def core_config(request) -> EverestEnvironmentCoreConfiguration:
@@ -43,6 +59,26 @@ def core_config(request) -> EverestEnvironmentCoreConfiguration:
     )
 
 
+@pytest.fixture
+def probe_module_config(request, core_config) -> Optional[EverestEnvironmentProbeModuleConfiguration]:
+    magic_probe_module_marker = request.node.get_closest_marker("magic_probe_module")
+    if magic_probe_module_marker:
+        configurator = _build_magic_probe_module_configurator(core_config)
+        return EverestEnvironmentProbeModuleConfiguration(
+            module_id=configurator.probe_module_id,
+            connections={k: [t[0] for t in requirements] for k, requirements in
+                         configurator.get_requirements().items()},
+            magic_probe_module_strict_mode=magic_probe_module_marker.kwargs.get("strict", False),
+            magic_probe_module_configurator=configurator
+        )
+    probe_module_marker = request.node.get_closest_marker("probe_module")
+    if probe_module_marker:
+        return EverestEnvironmentProbeModuleConfiguration(
+            **probe_module_marker.kwargs
+        )
+
+    return None
+
 @pytest.fixture
 def ocpp_config(request) -> Optional[EverestEnvironmentOCPPConfiguration]:
     return None

everest-testing/src/everest/testing/core_utils/magic_probe_module.py

@@ -1,65 +1,30 @@
-from pathlib import Path
-
 import pytest
 
-from ._magic_probe_module.parser.everest_interface_parser import EverestInterfaceParser as _EverestInterfaceParser
-from ._magic_probe_module.parser.everest_type_parser import EverestTypeParser as _EverestTypeParser
-from ._magic_probe_module.types.everest_config_schema import EverestConfigSchema as _EverestConfigSchema
 from ._configuration.everest_environment_setup import EverestEnvironmentCoreConfiguration, \
     EverestEnvironmentProbeModuleConfiguration
 from ._magic_probe_module.magic_probe_module import MagicProbeModule
-from ._magic_probe_module.magic_probe_module_configurator import MagicProbeModuleConfigurator
-from ._magic_probe_module.types.everest_module_manifest_schema import \
-    EverestModuleManifestSchema as _EverestModuleManifestSchema
-from ._magic_probe_module.value_generator import ValueGenerator
-import yaml
-
+from ._magic_probe_module.parser.everest_type_parser import EverestTypeParser as _EverestTypeParser
 from .everest_core import EverestCore
 
 
-@pytest.fixture()
-def magic_probe_module_configurator(core_config: EverestEnvironmentCoreConfiguration) -> MagicProbeModuleConfigurator:
-    everest_config = _EverestConfigSchema(**yaml.safe_load(core_config.template_everest_config_path.read_text()))
-
-    interfaces_directory = core_config.everest_core_path / "share" / "everest" / "interfaces"
-    modules_dir = core_config.everest_core_path / "libexec" / "everest" / "modules"
-
-    everest_interfaces = _EverestInterfaceParser().parse([interfaces_directory])
-
-    everest_manifests = {}
-    for f in modules_dir.glob("*"):
-        if (f / "manifest.yaml").exists():
-            everest_manifests[f.name] = _EverestModuleManifestSchema(
-                **yaml.safe_load((f / "manifest.yaml").read_text()))
-
-    configurator = MagicProbeModuleConfigurator(
-        everest_config=everest_config,
-        interfaces=everest_interfaces,
-        manifests=everest_manifests
-    )
-
-    return configurator
-
-
 @pytest.fixture
-def magic_probe_module(core_config: EverestEnvironmentCoreConfiguration, magic_probe_module_configurator: MagicProbeModuleConfigurator, everest_core: EverestCore) -> MagicProbeModule:
+def magic_probe_module(core_config: EverestEnvironmentCoreConfiguration,
+                       probe_module_config: EverestEnvironmentProbeModuleConfiguration | None,
+                       everest_core: EverestCore) -> MagicProbeModule:
+
+    if not probe_module_config or not probe_module_config.magic_probe_module_configurator:
+        raise AssertionError("Error: Usage of magic_probe_module fixture requires the @magic_probe_module marker or an override of the probe_module_config fixture.")
 
     types_directory = core_config.everest_core_path / "share" / "everest" / "types"
     everest_types = _EverestTypeParser().parse([types_directory])
 
+    magic_probe_module_configurator = probe_module_config.magic_probe_module_configurator
     return MagicProbeModule(
         interface_implementations=magic_probe_module_configurator.get_interface_implementations(),
         connections=magic_probe_module_configurator.get_requirements(),
         module_id=magic_probe_module_configurator.probe_module_id,
         session=everest_core.get_runtime_session(),
-        types=list(everest_types.values())
+        types=list(everest_types.values()),
+        strict_mode=probe_module_config.magic_probe_module_strict_mode
     )
 
-
-@pytest.fixture
-def probe_module_config(request, magic_probe_module_configurator) -> EverestEnvironmentProbeModuleConfiguration | None:
-    return EverestEnvironmentProbeModuleConfiguration(
-        module_id=magic_probe_module_configurator.probe_module_id,
-        connections={k: [t[0] for t in requirements] for k, requirements in
-                     magic_probe_module_configurator.get_requirements().items()}
-    )