Feature/715 combine component schemas and config in one file (#153)

* Ocpp 2.0.1 config is now in the same file as the component schema. Change some paths etc. * Start supporting component config instead of config.json for ocpp 2.0.1 tests * Test with component config changes and fix some bugs. Signed-off-by: Maaike Zijderveld, iolar <[email protected]> * ProbeModule: call commands with a 30s timeout This allows test cases to fail when EVerest abnormally closes * Expose OCPP 2.0.1 config with a fixture * Bump version to 0.4.0 Signed-off-by: Kai-Uwe Hermann <[email protected]> --------- Signed-off-by: Maaike Zijderveld, iolar <[email protected]> Signed-off-by: Kai-Uwe Hermann <[email protected]> Co-authored-by: Kai-Uwe Hermann <[email protected]>
EVerest · Nov 5, 2024 · 36bf7a2 · 36bf7a2
1 parent 3eb4c1b
commit 36bf7a2
Show file tree

Hide file tree

Showing 6 changed files with 107 additions and 58 deletions.
diff --git a/everest-testing/src/everest/testing/__init__.py b/everest-testing/src/everest/testing/__init__.py
@@ -1 +1 @@
-__version__="0.2.3"
+__version__="0.4.0"
diff --git a/...ils/_configuration/everest_configuration_strategies/ocpp_module_configuration_strategy.py b/...ils/_configuration/everest_configuration_strategies/ocpp_module_configuration_strategy.py
@@ -19,6 +19,7 @@ class OCPPModulePaths16(OCPPModuleConfigurationBase):
 
 @dataclass
 class OCPPModulePaths201(OCPPModuleConfigurationBase):
+    DeviceModelConfigPath: str
     CoreDatabasePath: str
     DeviceModelDatabasePath: str
 

diff --git a/everest-testing/src/everest/testing/core_utils/_configuration/everest_environment_setup.py b/everest-testing/src/everest/testing/core_utils/_configuration/everest_environment_setup.py
@@ -36,8 +36,8 @@ class EverestEnvironmentOCPPConfiguration:
     ocpp_module_id: str = "ocpp"
     template_ocpp_config: Optional[
         Path] = None  # Path for OCPP config to be used; if not provided, will be determined from everest config
-    device_model_schemas_path: Optional[
-        Path] = None  # Path of the OCPP device model json schemas. If not set, {libocpp_path} / 'config/v201/component_schemas' will  be used
+    device_model_component_config_path: Optional[
+        Path] = None  # Path of the OCPP device model json schemas. If not set, {libocpp_path} / 'config/v201/component_config' will  be used
     configuration_strategies: list[OCPPModuleConfigurationStrategy] | None = None
 
 
@@ -88,6 +88,7 @@ class EverestTestEnvironmentSetup:
     class _EverestEnvironmentTemporaryPaths:
         """ Paths of the temporary configuration files / data """
         certs_dir: Path  # used by both OCPP and evse security
+        ocpp_config_path: Path
         ocpp_config_file: Path
         ocpp_user_config_file: Path
         ocpp_database_dir: Path
@@ -113,6 +114,7 @@ def __init__(self,
             self._standalone_module = self._probe_config.module_id
         self._additional_everest_config_strategies = everest_config_strategies if everest_config_strategies else []
         self._everest_core = None
+        self._ocpp_configuration = None
 
     def setup_environment(self, tmp_path: Path):
 
@@ -127,7 +129,7 @@ def setup_environment(self, tmp_path: Path):
                                          tmp_path=tmp_path)
 
         if self._ocpp_config:
-            self._setup_libocpp_configuration(
+            self._ocpp_configuration = self._setup_libocpp_configuration(
                 temporary_paths=temporary_paths
             )
 
@@ -139,9 +141,18 @@ def everest_core(self) -> EverestCore:
         assert self._everest_core, "Everest Core not initialized; run 'setup_environment' first"
         return self._everest_core
 
+    @property
+    def ocpp_config(self):
+        return self._ocpp_configuration
+
     def _create_temporary_directory_structure(self, tmp_path: Path) -> _EverestEnvironmentTemporaryPaths:
         ocpp_config_dir = tmp_path / "ocpp_config"
         ocpp_config_dir.mkdir(exist_ok=True)
+        if self._ocpp_config.ocpp_version == OCPPVersion.ocpp201:
+            component_config_path_standardized = ocpp_config_dir / "component_config" / "standardized"
+            component_config_path_custom = ocpp_config_dir / "component_config" / "custom"
+            component_config_path_standardized.mkdir(parents=True, exist_ok=True)
+            component_config_path_custom.mkdir(parents=True, exist_ok=True)
         certs_dir = tmp_path / "certs"
         certs_dir.mkdir(exist_ok=True)
         ocpp_logs_dir = ocpp_config_dir / "logs"
@@ -153,6 +164,7 @@ def _create_temporary_directory_structure(self, tmp_path: Path) -> _EverestEnvir
         logging.info(f"temp ocpp config files directory: {ocpp_config_dir}")
 
         return self._EverestEnvironmentTemporaryPaths(
+            ocpp_config_path=ocpp_config_dir / "component_config",
             ocpp_config_file=ocpp_config_dir / "config.json",
             ocpp_user_config_file=ocpp_config_dir / "user_config.json",
             ocpp_database_dir=ocpp_config_dir,
@@ -173,6 +185,7 @@ def _create_ocpp_module_configuration_strategy(self,
             )
         elif self._ocpp_config.ocpp_version == OCPPVersion.ocpp201:
             ocpp_paths = OCPPModulePaths201(
+                DeviceModelConfigPath=str(temporary_paths.ocpp_config_path),
                 MessageLogPath=str(temporary_paths.ocpp_message_log_directory),
                 CoreDatabasePath=str(temporary_paths.ocpp_database_dir),
                 DeviceModelDatabasePath=str(temporary_paths.ocpp_database_dir / "device_model_storage.db"),
@@ -195,29 +208,21 @@ def _setup_libocpp_configuration(self, temporary_paths: _EverestEnvironmentTempo
         elif self._ocpp_config.ocpp_version == OCPPVersion.ocpp16:
             source_ocpp_config = self._determine_configured_charge_point_config_path_from_everest_config()
         elif self._ocpp_config.ocpp_version == OCPPVersion.ocpp201:
-            ocpp_dir = self._everest_core.prefix_path / "share/everest/modules/OCPP201"
-            source_ocpp_config = ocpp_dir / "config.json"
-
+            source_ocpp_config = self._ocpp_config.device_model_component_config_path \
+                if self._ocpp_config.device_model_component_config_path \
+                else self._ocpp_config.libocpp_path / 'config/v201/component_config'
 
-        liboccp_configuration_helper.generate_ocpp_config(
+        return liboccp_configuration_helper.generate_ocpp_config(
             central_system_port=self._ocpp_config.central_system_port,
             central_system_host=self._ocpp_config.central_system_host,
-            source_ocpp_config_file=source_ocpp_config,
-            target_ocpp_config_file=temporary_paths.ocpp_config_file,
+            source_ocpp_config_path=source_ocpp_config,
+            target_ocpp_config_path=temporary_paths.ocpp_config_file \
+                if self._ocpp_config.ocpp_version == OCPPVersion.ocpp16 \
+                else temporary_paths.ocpp_config_path,
             target_ocpp_user_config_file=temporary_paths.ocpp_user_config_file,
             configuration_strategies=self._ocpp_config.configuration_strategies
         )
 
-        if self._ocpp_config.ocpp_version == OCPPVersion.ocpp201:
-            liboccp_configuration_helper.create_temporary_ocpp_configuration_db(
-                libocpp_path=self._ocpp_config.libocpp_path,
-                device_model_schemas_path=self._ocpp_config.device_model_schemas_path \
-                    if self._ocpp_config.device_model_schemas_path \
-                    else self._ocpp_config.libocpp_path / 'config/v201/component_schemas',
-                ocpp_configuration_file=temporary_paths.ocpp_config_file,
-                target_directory=temporary_paths.ocpp_database_dir
-            )
-
     def _create_everest_configuration_strategies(self, temporary_paths: _EverestEnvironmentTemporaryPaths):
         configuration_strategies = []
         if self._ocpp_config:

diff --git a/...est-testing/src/everest/testing/core_utils/_configuration/libocpp_configuration_helper.py b/...est-testing/src/everest/testing/core_utils/_configuration/libocpp_configuration_helper.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import os
+from glob import glob
 import json
 import sys
 from abc import ABC, abstractmethod
@@ -34,30 +36,42 @@ class LibOCPPConfigurationHelperBase(ABC):
     """ Helper for parsing / adapting the LibOCPP configuration and dumping it a database file. """
 
     def generate_ocpp_config(self,
-                             target_ocpp_config_file: Path,
+                             target_ocpp_config_path: Path,
                              target_ocpp_user_config_file: Path,
-                             source_ocpp_config_file: Path,
+                             source_ocpp_config_path: Path,
                              central_system_host: str,
                              central_system_port: Union[str, int],
                              configuration_strategies: list[OCPPConfigAdjustmentStrategy] | None = None):
-        config = json.loads(source_ocpp_config_file.read_text())
+        config = self._get_config(source_ocpp_config_path)
 
         configuration_strategies = configuration_strategies if configuration_strategies else []
 
         for v in [self._get_default_strategy(central_system_port, central_system_host)] + configuration_strategies:
             config = v.adjust_ocpp_configuration(config)
 
-        with target_ocpp_config_file.open("w") as f:
-            json.dump(config, f)
+        self._store_config(config, target_ocpp_config_path)
         target_ocpp_user_config_file.write_text("{}")
 
+        return config
+
+    @abstractmethod
+    def _get_config(self, source_ocpp_config_path: Path):
+        pass
+
     @abstractmethod
     def _get_default_strategy(self, central_system_port: int | str,
                               central_system_host: str) -> OCPPConfigAdjustmentStrategy:
         pass
 
+    @abstractmethod
+    def _store_config(self, config, target_ocpp_config_file):
+        pass
+
 
 class LibOCPP16ConfigurationHelper(LibOCPPConfigurationHelperBase):
+    def _get_config(self, source_ocpp_config_path: Path):
+        return json.loads(source_ocpp_config_path.read_text())
+
     def _get_default_strategy(self, central_system_port, central_system_host):
         def adjust_ocpp_configuration(config: dict) -> dict:
             config = deepcopy(config)
@@ -68,6 +82,10 @@ def adjust_ocpp_configuration(config: dict) -> dict:
 
         return OCPPConfigAdjustmentStrategyWrapper(adjust_ocpp_configuration)
 
+    def _store_config(self, config, target_ocpp_config_file):
+        with target_ocpp_config_file.open("w") as f:
+            json.dump(config, f)
+
 
 class _OCPP201NetworkConnectionProfileAdjustment(OCPPConfigAdjustmentStrategy):
     """ Adjusts the OCPP 2.0.1 Network Connection Profile by injecting the right host, port and chargepoint id.
@@ -96,39 +114,50 @@ def adjust_ocpp_configuration(self, config: dict):
     @staticmethod
     def _get_value_from_v201_config(ocpp_config: json, component_name: str, variable_name: str,
                                     variable_attribute_type: str):
-        for component in ocpp_config:
-            if (component["name"] == component_name):
-                return component["variables"][variable_name]["attributes"][variable_attribute_type]
+        for (component, schema) in ocpp_config.items():
+            if component == component_name:
+                attributes = schema["properties"][variable_name]["attributes"]
+                for attribute in attributes:
+                    if attribute["type"] == variable_attribute_type:
+                        return attribute["value"]
 
     @staticmethod
     def _set_value_in_v201_config(ocpp_config: json, component_name: str, variable_name: str,
                                   variable_attribute_type: str,
                                   value: str):
-        for component in ocpp_config:
-            if (component["name"] == component_name):
-                component["variables"][variable_name]["attributes"][variable_attribute_type] = value
-                return
+        for (component, schema) in ocpp_config.items():
+            if component == component_name:
+                attributes = schema["properties"][variable_name]["attributes"]
+                for attribute in attributes:
+                    if attribute["type"] == variable_attribute_type:
+                        attribute["value"] = value
 
 
 class LibOCPP201ConfigurationHelper(LibOCPPConfigurationHelperBase):
 
+    def _get_config(self, source_ocpp_config_path: Path):
+        config = {}
+        file_list_standardized = glob(str(source_ocpp_config_path / "standardized" / "*.json"), recursive=False)
+        file_list_custom = glob(str(source_ocpp_config_path / "custom" / "*.json"), recursive=False)
+        file_list = file_list_standardized + file_list_custom
+        for file in file_list:
+            # Get component from file name
+            _, tail = os.path.split(file)
+            component_name, _ = os.path.splitext(tail)
+            # Store json in dict
+            with open(file) as f:
+                config[component_name] = json.load(f)
+        return config
+
     def _get_default_strategy(self, central_system_port: int | str,
                               central_system_host: str) -> OCPPConfigAdjustmentStrategy:
         return _OCPP201NetworkConnectionProfileAdjustment(central_system_port, central_system_host)
 
-    @staticmethod
-    def create_temporary_ocpp_configuration_db(libocpp_path: Path,
-                                               device_model_schemas_path: Path,
-                                               ocpp_configuration_file: Path,
-                                               target_directory: Path):
-        import_path = libocpp_path / "config/v201"
-        sys.path.append(str(import_path))
-        from init_device_model_db import DeviceModelDatabaseInitializer
-
-        database_file = target_directory / 'device_model_storage.db'
-        database_initializer = DeviceModelDatabaseInitializer(database_file)
-
-        database_initializer.initialize_database(schemas_path=device_model_schemas_path)
-        database_initializer.insert_config_and_default_values(
-            config_file=ocpp_configuration_file,
-            schemas_path=device_model_schemas_path)
+    def _store_config(self, config, target_ocpp_config_path):
+        # Just store all in the 'standardized' folder
+        path = target_ocpp_config_path / "standardized"
+        for key, value in config.items():
+            file_name = path / (key + '.json')
+            file_name.parent.mkdir(parents=True, exist_ok=True)
+            with file_name.open("w+") as f:
+                json.dump(value, f)
diff --git a/everest-testing/src/everest/testing/core_utils/fixtures.py b/everest-testing/src/everest/testing/core_utils/fixtures.py
@@ -75,20 +75,17 @@ def everest_config_strategies(request) -> list[EverestConfigAdjustmentStrategy]:
     return additional_configuration_strategies
 
 @pytest.fixture
-def everest_core(request,
+def everest_environment(request,
                  tmp_path,
                  core_config: EverestEnvironmentCoreConfiguration,
                  ocpp_config: Optional[EverestEnvironmentOCPPConfiguration],
                  probe_module_config: Optional[EverestEnvironmentProbeModuleConfiguration],
                  evse_security_config: Optional[EverestEnvironmentEvseSecurityConfiguration],
                  persistent_store_config: Optional[EverestEnvironmentPersistentStoreConfiguration],
                  everest_config_strategies
-                 ) -> EverestCore:
-    """Fixture that can be used to start and stop everest-core"""
-
+                 ):
     standalone_module_marker = request.node.get_closest_marker('standalone_module')
 
-
     environment_setup = EverestTestEnvironmentSetup(
         core_config=core_config,
         ocpp_config=ocpp_config,
@@ -100,11 +97,23 @@ def everest_core(request,
     )
 
     environment_setup.setup_environment(tmp_path=tmp_path)
-    yield environment_setup.everest_core
+
+    yield environment_setup
+
+@pytest.fixture
+def everest_core(request,
+                 everest_environment
+                 )-> EverestCore:
+    """Fixture that can be used to start and stop everest-core"""
+
+    yield everest_environment.everest_core
 
     # FIXME (aw): proper life time management, shouldn't the fixure start and stop?
-    environment_setup.everest_core.stop()
+    everest_environment.everest_core.stop()
 
+@pytest.fixture
+def ocpp_configuration(everest_environment):
+    yield everest_environment.ocpp_config
 
 @pytest.fixture
 def test_controller(request, tmp_path, everest_core) -> EverestTestController:

diff --git a/everest-testing/src/everest/testing/core_utils/probe_module.py b/everest-testing/src/everest/testing/core_utils/probe_module.py
@@ -51,9 +51,15 @@ async def call_command(self, connection_id: str, command_name: str, args: dict)
 
         interface = self._setup.connections[connection_id][0]
         try:
-            return await asyncio.to_thread(lambda: self._mod.call_command(interface, command_name, args))
+            async with asyncio.timeout(30):
+                return await asyncio.to_thread(lambda: self._mod.call_command(interface, command_name, args))
+        except TimeoutError as e:
+            error_message = f"Timeout in calling {connection_id}.{command_name}: {type(e)}: {e}. This might be caused by the other module/EVerest exiting abnormally."
+            logging.error(error_message)
+            raise RuntimeError(error_message)
         except Exception as e:
-            logging.info(f"Exception in calling {connection_id}.{command_name}: {type(e)}: {e}")
+            logging.info(
+                f"Exception in calling {connection_id}.{command_name}: {type(e)}: {e}")
 
     def implement_command(self, implementation_id: str, command_name: str, handler: Callable[[dict], Any]):
         """
@@ -118,7 +124,6 @@ async def wait_for_event(self, timeout: float):
         if not self._ready_event.is_set():
             raise TimeoutError("Waiting for ready: timeout")
 
-
     async def wait_to_be_ready(self, timeout=3.0):
         """
         Convenience method which allows you to wait until the _ready() callback is triggered (i.e. until EVerest is up and running)