diff --git a/everest-testing/src/everest/testing/core_utils/_configuration/everest_environment_setup.py b/everest-testing/src/everest/testing/core_utils/_configuration/everest_environment_setup.py index 4d8280d5..de5ab9c4 100644 --- a/everest-testing/src/everest/testing/core_utils/_configuration/everest_environment_setup.py +++ b/everest-testing/src/everest/testing/core_utils/_configuration/everest_environment_setup.py @@ -25,6 +25,7 @@ ProbeModuleConfigurationStrategy from .libocpp_configuration_helper import \ LibOCPP201ConfigurationHelper, LibOCPP16ConfigurationHelper +from .._magic_probe_module.magic_probe_module_configurator import MagicProbeModuleConfigurator @dataclass @@ -68,6 +69,8 @@ class EverestEnvironmentCoreConfiguration: class EverestEnvironmentProbeModuleConfiguration: connections: Dict[str, List[Requirement]] = field(default_factory=dict) module_id: str = "probe" + magic_probe_module_strict_mode: bool = False, + magic_probe_module_configurator: Optional[MagicProbeModuleConfigurator] = None class EverestTestEnvironmentSetup: @@ -225,6 +228,8 @@ def _create_everest_configuration_strategies(self, temporary_paths: _EverestEnvi configuration_strategies.append( ProbeModuleConfigurationStrategy(connections=self._probe_config.connections, module_id=self._probe_config.module_id)) + if self._probe_config.magic_probe_module_configurator: + configuration_strategies.append(self._probe_config.magic_probe_module_configurator.get_configuration_adjustment_strategy()) if self._evse_security_config: configuration_strategies.append( diff --git a/everest-testing/src/everest/testing/core_utils/_magic_probe_module/README.md b/everest-testing/src/everest/testing/core_utils/_magic_probe_module/README.md new file mode 100644 index 00000000..b3abb81d --- /dev/null +++ b/everest-testing/src/everest/testing/core_utils/_magic_probe_module/README.md @@ -0,0 +1,136 @@ +# The Magic Probe Module + +## Motivation + +The "pure" Probe Module comes with the issue of high configuration effort. To "quickly" test another's module interface, +a lot of configuration is required. Furthermore, if a probe module's commands are called that are not explicitly implemented, +the process hangs without further notification. + +The idea behind the "magic" probe module is similar to the distinction of the "Mock" and "MagicMock" classes in +Python unittests: The latter "out of the box" provides all magic methods (for comparisons etc.) and requires less configuration. + +The Magic Probe Module should configure itself automatically as much as possible and mitigate the drawbacks of the default +one. + + +## Core Concepts + +The Magic Probe Module: +- Gets activated by the marker `@magic_probe_module` and this then available via the `magic_probe_module` fixture. +- Is automatically configured to fulfill _any_ requirement that is not fulfilled in the provided Everest configuration. +- Also detects existing requirement fulfillments in the provided Everest configuration (in case certain implementation ids shall be useed) +- Automatically implements _any_ command and wraps it into mock calls. The `implement_command` method of the Magic Probe Module +allows to set the return value or side effect of each command. This can be done even after module startup. Per default, +it provides a auto-generated value. Can be set (via the strict parameter of the `@magic_probe_module` to _strict_ mode to +raise an Exception per default instead) +- Possesses a value generator that is capable of generating any EVerest type (as far as specified in the Json Schema; works most of the timel™) + +## Requirements + +The magic probe module currently requires packages: +- pydantic >= 2 +- rstr + +## Usage Examples + +Given the config: + +```yaml +active_modules: + ocpp: + module: OCPP201 + config_module: + ChargePointConfigPath: config.json + EnableExternalWebsocketControl: true + connections: {} +x-module-layout: {} + + +``` + +the following example (assuming correct imports) works: + +```python + +@pytest.mark.asyncio +@pytest.mark.ocpp_version("ocpp2.0.1") +@pytest.mark.everest_core_config("everest-config-ocpp201-magic-probe-module.yaml") +@pytest.mark.magic_probe_module() +async def test_against_occp(central_system: CentralSystem, + test_controller, + magic_probe_module, + test_utility: TestUtility): + test_controller.start() + magic_probe_module.start() + await magic_probe_module.wait_to_be_ready() + await central_system.wait_for_chargepoint() + + # get all implementations for a certain interface + evse_managers = [impl for impl, intf in magic_probe_module.get_interface_implementations().items() if + intf.interface == "evse_manager"] + + for evse_manager_implementation in evse_managers: + magic_probe_module.publish_variable(evse_manager_implementation, "iso15118_certificate_request", + {"exiRequest": "bla", + "iso15118SchemaVersion": "mock_iso15118_schema_version", + "certificateAction": "ba"}) + await asyncio.sleep(1) + # The OCPP module should have called the "enable" endpoint of the evse_manager, this command is automatically implemented + magic_probe_module.implementation_mocks[evse_manager_implementation].enable.assert_called_with( + {"connector_id": ANY}) + +``` + +Note that under the hood the Magic Probe Module will create the following configuration that EVerest is started with: +```yaml +active_modules: + ocpp: + config_module: + CertsPath: /tmp/sharedtmp/pytest/test_against_occp0/certs + ChargePointConfigPath: /tmp/sharedtmp/pytest/test_against_occp0/ocpp_config/config.json + CoreDatabasePath: /tmp/sharedtmp/pytest/test_against_occp0/ocpp_config + DeviceModelDatabasePath: /tmp/sharedtmp/pytest/test_against_occp0/ocpp_config/device_model_storage.db + EnableExternalWebsocketControl: true + MessageLogPath: /tmp/sharedtmp/pytest/test_against_occp0/ocpp_config/logs + connections: + evse_manager: + - implementation_id: ProbeModuleConnectorA + module_id: probe + kvs: + - implementation_id: kvs + module_id: probe + security: + - implementation_id: evse_security + module_id: probe + system: + - implementation_id: system + module_id: probe + module: OCPP201 + probe: + connections: + auth_token_provider: + - implementation_id: auth_provider + module_id: ocpp + auth_token_validator: + - implementation_id: auth_validator + module_id: ocpp + empty: + - implementation_id: main + module_id: ocpp + ocpp_data_transfer: + - implementation_id: data_transfer + module_id: ocpp + module: ProbeModule +settings: + controller_port: 0 + mqtt_everest_prefix: everest_5b6796904fe04146b0b79dd07de69c65 + mqtt_external_prefix: external_5b6796904fe04146b0b79dd07de69c65 + telemetry_prefix: telemetry_5b6796904fe04146b0b79dd07de69c65 +x-module-layout: {} + +``` + +## Todos + +- Issue with invalid values (e.g. Connector ids must be sequential) +- Downgrade to Pydantic 1 \ No newline at end of file diff --git a/everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module.py b/everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module.py index c587ddd4..13afc91c 100644 --- a/everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module.py +++ b/everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module.py @@ -8,6 +8,7 @@ from .types.everest_command import EverestCommand from .types.everest_interface import EverestInterface from .types.everest_type import EverestType +from .types.json_schema_models import JsonSchema from .value_generator import ValueGenerator class MagicProbeModule(ProbeModule): @@ -36,6 +37,12 @@ def __init__(self, self._implementation_mocks: dict[str, Mock] = {} self._init_commands() + async def wait_to_be_ready(self, timeout=3): + await super().wait_to_be_ready(timeout=timeout) + for interface_implementation in self._interface_implementations: + self.publish_variable(interface_implementation,"ready", True) + + def implement_command(self, implementation_id: str, command_name: str, handler: Callable[[dict], Any]): getattr(self._implementation_mocks[implementation_id], command_name).side_effect = handler @@ -95,10 +102,16 @@ def _handler(*args, **kwargs): return res elif command.result: if self._strict_mode: - raise NotImplementedError(f"MagicProbeModule command {implementation_id} / {command} not implemented") + error = f"MagicProbeModule command {implementation_id} / {command} not implemented - aborting test" + logging.error(error) + raise NotImplementedError(error) logging.warning(f"MagicProbeModule command {implementation_id} / {command} not implemented - returning auto-generated value!") return self._value_generator.generate(command.result) else: return None return _handler + + + def get_value(self, what: EverestType | JsonSchema | str): + self._value_generator.generate(what) \ No newline at end of file diff --git a/everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module_configurator.py b/everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module_configurator.py index b5bf9ac9..b1385a02 100644 --- a/everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module_configurator.py +++ b/everest-testing/src/everest/testing/core_utils/_magic_probe_module/magic_probe_module_configurator.py @@ -1,14 +1,12 @@ import logging from dataclasses import dataclass -from typing import Dict +from everest.testing.core_utils import EverestConfigAdjustmentStrategy from everest.testing.core_utils.common import Requirement - from .magic_probe_module_config_strategy import MagicProbeModuleConfigurationStrategy from .types.everest_config_schema import EverestConfigSchema from .types.everest_interface import EverestInterface from .types.everest_module_manifest_schema import EverestModuleManifestSchema -from everest.testing.core_utils import EverestConfigAdjustmentStrategy @dataclass @@ -168,3 +166,5 @@ def _determine_probe_module_requirements(self) -> dict[str, dict[str, list[tuple self._interfaces_by_name[provides.interface]) ) return probe_module_requirements + + diff --git a/everest-testing/src/everest/testing/core_utils/everest_core.py b/everest-testing/src/everest/testing/core_utils/everest_core.py index 8458dd41..90501eef 100644 --- a/everest-testing/src/everest/testing/core_utils/everest_core.py +++ b/everest-testing/src/everest/testing/core_utils/everest_core.py @@ -193,13 +193,17 @@ def start(self, standalone_module: Optional[Union[str, List[str]]] = None, test_ expected_status = 'ALL_MODULES_STARTED' if standalone_module == None else 'WAITING_FOR_STANDALONE_MODULES' - status = self.status_listener.wait_for_status(STARTUP_TIMEOUT, [expected_status]) - if status == None or len(status) == 0: - raise TimeoutError("Timeout while waiting for EVerest to start") - - logging.info("EVerest has started") - if expected_status == 'ALL_MODULES_STARTED': + if standalone_module and set(standalone_module) - {"probe"}: + time.sleep(25) self.all_modules_started_event.set() + else: + status = self.status_listener.wait_for_status(STARTUP_TIMEOUT, [expected_status]) + if status == None or len(status) == 0: + raise TimeoutError("Timeout while waiting for EVerest to start") + + logging.info("EVerest has started") + if expected_status == 'ALL_MODULES_STARTED': + self.all_modules_started_event.set() def read_everest_log(self): while self.process.poll() == None: diff --git a/everest-testing/src/everest/testing/core_utils/fixtures.py b/everest-testing/src/everest/testing/core_utils/fixtures.py index dc65c808..0e549bcd 100644 --- a/everest-testing/src/everest/testing/core_utils/fixtures.py +++ b/everest-testing/src/everest/testing/core_utils/fixtures.py @@ -14,17 +14,33 @@ from everest.testing.core_utils.controller.everest_test_controller import EverestTestController from everest.testing.core_utils.everest_core import EverestCore +from ._magic_probe_module.magic_probe_module_configurator import MagicProbeModuleConfigurator +from ._magic_probe_module.parser.everest_interface_parser import EverestInterfaceParser +from ._magic_probe_module.types.everest_config_schema import EverestConfigSchema as _EverestConfigSchema +from ._magic_probe_module.types.everest_module_manifest_schema import EverestModuleManifestSchema +import yaml + +def _build_magic_probe_module_configurator(core_config: EverestEnvironmentCoreConfiguration) -> MagicProbeModuleConfigurator: + everest_config = _EverestConfigSchema(**yaml.safe_load(core_config.template_everest_config_path.read_text())) + + interfaces_directory = core_config.everest_core_path / "share" / "everest" / "interfaces" + modules_dir = core_config.everest_core_path / "libexec" / "everest" / "modules" + + everest_interfaces = EverestInterfaceParser().parse([interfaces_directory]) + + everest_manifests = {} + for f in modules_dir.glob("*"): + if (f / "manifest.yaml").exists(): + everest_manifests[f.name] = EverestModuleManifestSchema( + **yaml.safe_load((f / "manifest.yaml").read_text())) + + configurator = MagicProbeModuleConfigurator( + everest_config=everest_config, + interfaces=everest_interfaces, + manifests=everest_manifests + ) -@pytest.fixture -def probe_module_config(request) -> Optional[EverestEnvironmentProbeModuleConfiguration]: - marker = request.node.get_closest_marker("probe_module") - if marker: - return EverestEnvironmentProbeModuleConfiguration( - **marker.kwargs - ) - - return None - + return configurator @pytest.fixture def core_config(request) -> EverestEnvironmentCoreConfiguration: @@ -43,6 +59,26 @@ def core_config(request) -> EverestEnvironmentCoreConfiguration: ) +@pytest.fixture +def probe_module_config(request, core_config) -> Optional[EverestEnvironmentProbeModuleConfiguration]: + magic_probe_module_marker = request.node.get_closest_marker("magic_probe_module") + if magic_probe_module_marker: + configurator = _build_magic_probe_module_configurator(core_config) + return EverestEnvironmentProbeModuleConfiguration( + module_id=configurator.probe_module_id, + connections={k: [t[0] for t in requirements] for k, requirements in + configurator.get_requirements().items()}, + magic_probe_module_strict_mode=magic_probe_module_marker.kwargs.get("strict", False), + magic_probe_module_configurator=configurator + ) + probe_module_marker = request.node.get_closest_marker("probe_module") + if probe_module_marker: + return EverestEnvironmentProbeModuleConfiguration( + **probe_module_marker.kwargs + ) + + return None + @pytest.fixture def ocpp_config(request) -> Optional[EverestEnvironmentOCPPConfiguration]: return None diff --git a/everest-testing/src/everest/testing/core_utils/magic_probe_module.py b/everest-testing/src/everest/testing/core_utils/magic_probe_module.py index 2655df78..7d4666aa 100644 --- a/everest-testing/src/everest/testing/core_utils/magic_probe_module.py +++ b/everest-testing/src/everest/testing/core_utils/magic_probe_module.py @@ -1,65 +1,30 @@ -from pathlib import Path - import pytest -from ._magic_probe_module.parser.everest_interface_parser import EverestInterfaceParser as _EverestInterfaceParser -from ._magic_probe_module.parser.everest_type_parser import EverestTypeParser as _EverestTypeParser -from ._magic_probe_module.types.everest_config_schema import EverestConfigSchema as _EverestConfigSchema from ._configuration.everest_environment_setup import EverestEnvironmentCoreConfiguration, \ EverestEnvironmentProbeModuleConfiguration from ._magic_probe_module.magic_probe_module import MagicProbeModule -from ._magic_probe_module.magic_probe_module_configurator import MagicProbeModuleConfigurator -from ._magic_probe_module.types.everest_module_manifest_schema import \ - EverestModuleManifestSchema as _EverestModuleManifestSchema -from ._magic_probe_module.value_generator import ValueGenerator -import yaml - +from ._magic_probe_module.parser.everest_type_parser import EverestTypeParser as _EverestTypeParser from .everest_core import EverestCore -@pytest.fixture() -def magic_probe_module_configurator(core_config: EverestEnvironmentCoreConfiguration) -> MagicProbeModuleConfigurator: - everest_config = _EverestConfigSchema(**yaml.safe_load(core_config.template_everest_config_path.read_text())) - - interfaces_directory = core_config.everest_core_path / "share" / "everest" / "interfaces" - modules_dir = core_config.everest_core_path / "libexec" / "everest" / "modules" - - everest_interfaces = _EverestInterfaceParser().parse([interfaces_directory]) - - everest_manifests = {} - for f in modules_dir.glob("*"): - if (f / "manifest.yaml").exists(): - everest_manifests[f.name] = _EverestModuleManifestSchema( - **yaml.safe_load((f / "manifest.yaml").read_text())) - - configurator = MagicProbeModuleConfigurator( - everest_config=everest_config, - interfaces=everest_interfaces, - manifests=everest_manifests - ) - - return configurator - - @pytest.fixture -def magic_probe_module(core_config: EverestEnvironmentCoreConfiguration, magic_probe_module_configurator: MagicProbeModuleConfigurator, everest_core: EverestCore) -> MagicProbeModule: +def magic_probe_module(core_config: EverestEnvironmentCoreConfiguration, + probe_module_config: EverestEnvironmentProbeModuleConfiguration | None, + everest_core: EverestCore) -> MagicProbeModule: + + if not probe_module_config or not probe_module_config.magic_probe_module_configurator: + raise AssertionError("Error: Usage of magic_probe_module fixture requires the @magic_probe_module marker or an override of the probe_module_config fixture.") types_directory = core_config.everest_core_path / "share" / "everest" / "types" everest_types = _EverestTypeParser().parse([types_directory]) + magic_probe_module_configurator = probe_module_config.magic_probe_module_configurator return MagicProbeModule( interface_implementations=magic_probe_module_configurator.get_interface_implementations(), connections=magic_probe_module_configurator.get_requirements(), module_id=magic_probe_module_configurator.probe_module_id, session=everest_core.get_runtime_session(), - types=list(everest_types.values()) + types=list(everest_types.values()), + strict_mode=probe_module_config.magic_probe_module_strict_mode ) - -@pytest.fixture -def probe_module_config(request, magic_probe_module_configurator) -> EverestEnvironmentProbeModuleConfiguration | None: - return EverestEnvironmentProbeModuleConfiguration( - module_id=magic_probe_module_configurator.probe_module_id, - connections={k: [t[0] for t in requirements] for k, requirements in - magic_probe_module_configurator.get_requirements().items()} - )