From b52f79ad4f3be847c991411e863cf86d2b943dbd Mon Sep 17 00:00:00 2001 From: Dariusz Suchojad Date: Sat, 9 Dec 2023 14:32:36 +0100 Subject: [PATCH] GH #755 - Extending hot-deployment patterns. --- code/zato-common/src/zato/common/api.py | 3 +++ .../src/zato/common/hot_deploy_.py | 4 ++++ code/zato-common/src/zato/common/util/api.py | 9 +++++++++ code/zato-common/src/zato/common/util/env.py | 19 +++++++++++++++++++ .../src/zato/common/util/hot_deploy_.py | 17 ++++++++++++++++- .../src/zato/server/base/parallel/__init__.py | 15 ++++++++++----- 6 files changed, 61 insertions(+), 6 deletions(-) diff --git a/code/zato-common/src/zato/common/api.py b/code/zato-common/src/zato/common/api.py index 38eba7d719..3dd31a11d0 100644 --- a/code/zato-common/src/zato/common/api.py +++ b/code/zato-common/src/zato/common/api.py @@ -1797,6 +1797,9 @@ class HotDeploy: Enmasse_File_Pattern = 'enmasse' Default_Patterns = [User_Conf_Directory, Enmasse_File_Pattern] + class Env: + Pickup_Patterns = 'Zato_Hot_Deploy_Pickup_Patterns' + # ################################################################################################################################ # ################################################################################################################################ diff --git a/code/zato-common/src/zato/common/hot_deploy_.py b/code/zato-common/src/zato/common/hot_deploy_.py index 45f0391661..2cb684ad2e 100644 --- a/code/zato-common/src/zato/common/hot_deploy_.py +++ b/code/zato-common/src/zato/common/hot_deploy_.py @@ -26,6 +26,10 @@ class HotDeployProject: # ################################################################################################################################ # ################################################################################################################################ +# +# These are default patterns that can be extended in runtime +# via the HotDeploy.Env.Pickup_Patterns environment variable. +# pickup_order_patterns = [ ' common*/**', diff --git a/code/zato-common/src/zato/common/util/api.py b/code/zato-common/src/zato/common/util/api.py index 1f33557b30..09ed6d507a 100644 --- a/code/zato-common/src/zato/common/util/api.py +++ b/code/zato-common/src/zato/common/util/api.py @@ -2133,3 +2133,12 @@ def load_ipc_pid_port(cluster_name:'str', server_name:'str', pid:'int') -> 'int' return out # ################################################################################################################################ + +def make_list_from_string_list(value:'str', separator:'str') -> 'strlist': + + value = value.split(separator) # type: ignore + value = [elem.strip() for elem in value if elem] # type: ignore + + return value + +# ################################################################################################################################ diff --git a/code/zato-common/src/zato/common/util/env.py b/code/zato-common/src/zato/common/util/env.py index aba8629d48..d75ecca33a 100644 --- a/code/zato-common/src/zato/common/util/env.py +++ b/code/zato-common/src/zato/common/util/env.py @@ -82,5 +82,24 @@ def populate_environment_from_file(env_path:'str', *, to_delete:'strlistnone'=No # We are ready to return our response now return out +# ################################################################################################################################ + +def get_list_from_environment(key:'str', separator:'str') -> 'strlist': + + # stdlib + import os + + # Zato + from zato.common.util.api import make_list_from_string_list + + # Our value to produce + out = [] + + if value := os.environ.get(key): + value = make_list_from_string_list(value, separator) + out.extend(value) + + return out + # ################################################################################################################################ # ################################################################################################################################ diff --git a/code/zato-common/src/zato/common/util/hot_deploy_.py b/code/zato-common/src/zato/common/util/hot_deploy_.py index 08f3ec227f..fbc6bd7176 100644 --- a/code/zato-common/src/zato/common/util/hot_deploy_.py +++ b/code/zato-common/src/zato/common/util/hot_deploy_.py @@ -7,12 +7,14 @@ """ # stdlib +from logging import getLogger from pathlib import Path # Zato from zato.common.api import HotDeploy from zato.common.hot_deploy_ import HotDeployProject, pickup_order_patterns from zato.common.typing_ import cast_ +from zato.common.util.env import get_list_from_environment from zato.common.util.file_system import resolve_path # ################################################################################################################################ @@ -24,6 +26,11 @@ # ################################################################################################################################ # ################################################################################################################################ +logger = getLogger(__name__) + +# ################################################################################################################################ +# ################################################################################################################################ + def get_project_info( root, # type: str src_dir_name, # type: str @@ -55,8 +62,16 @@ def get_project_info( # .. this is what will be added to sys.path by a starting server .. project.sys_path_entry = item + # .. make use of the default patterns .. + patterns = pickup_order_patterns[:] + + # .. append any extra patterns found in the environment .. + if extra_patterns := get_list_from_environment(HotDeploy.Env.Pickup_Patterns, ','): + logger.info('Found extra hot-deployment patterns via %s -> %s', HotDeploy.Env.Pickup_Patterns, extra_patterns) + patterns.extend(extra_patterns) + # .. look up all the directories to pick up from .. - for pattern in pickup_order_patterns: + for pattern in patterns: # .. remove any potential whitespace .. pattern = pattern.strip() diff --git a/code/zato-server/src/zato/server/base/parallel/__init__.py b/code/zato-server/src/zato/server/base/parallel/__init__.py index cad40780a1..58c84b0ab3 100644 --- a/code/zato-server/src/zato/server/base/parallel/__init__.py +++ b/code/zato-server/src/zato/server/base/parallel/__init__.py @@ -55,8 +55,8 @@ from zato.common.rate_limiting import RateLimiting from zato.common.typing_ import cast_, intnone, optional from zato.common.util.api import absolutize, get_config_from_file, get_kvdb_config_for_log, get_user_config_name, \ - fs_safe_name, hot_deploy, invoke_startup_services as _invoke_startup_services, new_cid, register_diag_handlers, \ - save_ipc_pid_port, spawn_greenlet, StaticConfig + fs_safe_name, hot_deploy, invoke_startup_services as _invoke_startup_services, make_list_from_string_list, new_cid, \ + register_diag_handlers, save_ipc_pid_port, spawn_greenlet, StaticConfig from zato.common.util.env import populate_environment_from_file from zato.common.util.file_transfer import path_string_list_to_list from zato.common.util.hot_deploy_ import extract_pickup_from_items @@ -430,10 +430,16 @@ def import_initial_services_jobs() -> 'anyset': locally_deployed.extend(user_defined_deployed) len_user_defined_deployed = len(user_defined_deployed) - suffix = ' ' if len_user_defined_deployed == 1 else 's ' + suffix = '' if len_user_defined_deployed == 1 else 's' + # This will be always logged .. logger.info('Deployed %d user-defined service%s (%s)', len_user_defined_deployed, suffix, self.name) + # .. whereas details are optional. + if asbool(os.environ.get('Zato_Log_User_Services_Deployed', False)): + for item in sorted(elem.name for elem in user_defined_deployed): + logger.info('Deployed user service: %s', item) + return set(locally_deployed) lock_name = '{}{}:{}'.format( @@ -537,8 +543,7 @@ def add_pickup_conf_from_local_path(self, paths:'str', source:'str', path_patter logger.info(msg) # .. support multiple entries .. - paths = paths.split(':') # type: ignore - paths = [elem.strip() for elem in paths if elem] # type: ignore + paths = make_list_from_string_list(paths, ':') # .. add the actual configuration .. for path in paths: