diff --git a/components/tools/OmeroPy/src/omero/config.py b/components/tools/OmeroPy/src/omero/config.py index 247a567dba9..b7677b00135 100644 --- a/components/tools/OmeroPy/src/omero/config.py +++ b/components/tools/OmeroPy/src/omero/config.py @@ -30,6 +30,11 @@ ) from omero_ext import portalocker import json +try: + import yaml + YAML_ENABLED = True +except ImportError: + YAML_ENABLED = False class Environment(object): @@ -439,3 +444,110 @@ def __delitem__(self, key): to_remove.append(p) for x in to_remove: props.remove(x) + + +def load_json_config_dir(configdir): + """ + Load a directory of .json, .yaml or .yml configuration files. + """ + files = (f for f in os.listdir(configdir) if + not f.startswith('.') and + os.path.splitext(f)[1].lower() in ('.json', '.yaml', '.yml')) + cset, cappend = load_json_configs( + os.path.join(configdir, f) for f in sorted(files)) + return cset, cappend + + +def load_json_configs(configfiles): + """ + Load a set of configuration files in the given order, and return two + dictionaries: + - values to be set + - values to be appended + + The caller should combine these two dictionaries appropriately. + Typically properties to be `set` will be processed first to allow existing + defaults to be unset if necessary, followed by processing of properties to + `append`. + + Each file is a single JSON or YAML dictionary (set of key-value pairs). + A special key `_mode` can take the value `"set"` (default) or `"append"` + which determines whether the properties will be added to the `set` or + `append` config dictionary. + + In `set` mode duplicate properties will overwrite existing ones, this is + equivalent to `omero config set` with the following rules: + - a null value will unset the configuration key + - otherwise the configuration key will be set to the value + + In `append` mode the value of each field must be a list or dictionary, it + is not possible to append a scalar. + - If the value is a list all values in the list will be appended to the + current property value + - If the value is a dictionary it will be merged non-recursively to the + all values in the list will be appended to the + current property value + + :param configfiles [str]: List of JSON or YAML (if module is available) + filepaths to load + :return (dict, dict): Tuple of config-set and config-append properties + """ + cset = {} + cappend = {} + + for f in configfiles: + with open(f) as fh: + try: + if f.lower().endswith('.yaml') or f.lower().endswith('.yml'): + if not YAML_ENABLED: + raise Exception( + 'PyYAML module required to load {}'.format(f)) + d = yaml.load(fh) + else: + d = json.load(fh) + except Exception as e: + raise Exception('Failed to load file {}: {}'.format(f, e)) + mode = d.pop('_mode', 'set') + for k, v in d.items(): + if mode == 'set': + _json_config_set(cset, k, v) + elif mode == 'append': + _json_config_append(cappend, k, v) + else: + raise Exception( + 'Invalid configuration mode: {}'.format(mode)) + + return cset, cappend + + +def _json_config_set(config, k, v): + if v is None: + config.pop(k, None) + else: + config[k] = v + + +def _json_config_append(config, k, v): + if isinstance(v, list): + if k not in config: + config[k] = v + else: + try: + config[k].extend(v) + except AttributeError: + raise Exception( + 'Incompatible types for append key {}: {} {}'.format( + k, config[k], v)) + elif isinstance(v, dict): + if k not in config: + config[k] = v + else: + try: + config[k].update(v) + except AttributeError: + raise Exception( + 'Incompatible types for append key {}: {} {}'.format( + k, config[k], v)) + else: + raise Exception( + 'Append requires a list or dictionary value: {}: {}'.format(k, v)) diff --git a/components/tools/OmeroPy/test/unit/test_config.py b/components/tools/OmeroPy/test/unit/test_config.py index a868914c88b..acf33a95f97 100644 --- a/components/tools/OmeroPy/test/unit/test_config.py +++ b/components/tools/OmeroPy/test/unit/test_config.py @@ -12,10 +12,18 @@ import os import errno import pytest -from omero.config import ConfigXml, xml +from omero.config import ( + ConfigXml, + xml, + _json_config_append, + _json_config_set, + load_json_config_dir, + load_json_configs, +) from omero.util.temp_files import create_path from omero_ext import portalocker +import json from xml.etree.ElementTree import XML, Element, SubElement, tostring @@ -355,3 +363,126 @@ def testCannotRead(self): with pytest.raises(IOError) as excinfo: ConfigXml(str(p)).close() assert excinfo.value.errno == errno.EACCES + + +class TestJsonConfig(object): + + def test_json_config_set(self): + cfg = {} + _json_config_set(cfg, 'a', 'a1') + assert cfg == {'a': 'a1'} + _json_config_set(cfg, 'b', 'b2') + assert cfg == {'a': 'a1', 'b': 'b2'} + _json_config_set(cfg, 'a', 'a11') + assert cfg == {'a': 'a11', 'b': 'b2'} + + def test_json_config_append_invalid(self): + cfg = { + 'a': 1, + 'b': [2], + 'c': {'c3': 3}, + } + with pytest.raises(Exception): + _json_config_append(cfg, 'a', 'not-list-or-dict') + with pytest.raises(Exception): + _json_config_append(cfg, 'b', 'not-list-or-dict') + with pytest.raises(Exception): + _json_config_append(cfg, 'c', 'not-list-or-dict') + + with pytest.raises(Exception): + _json_config_append(cfg, 'b', {'not': 'list'}) + with pytest.raises(Exception): + _json_config_append(cfg, 'c', ['not', 'dict']) + + def test_json_config_append_list(self): + cfg = {} + _json_config_append(cfg, 'b', [2]) + assert cfg == {'b': [2]} + _json_config_append(cfg, 'b', [3]) + assert cfg == {'b': [2, 3]} + _json_config_append(cfg, 'b', [4, 5]) + assert cfg == {'b': [2, 3, 4, 5]} + + def test_json_config_append_dict(self): + cfg = {} + _json_config_append(cfg, 'c', {'c3': 3}) + assert cfg == {'c': {'c3': 3}} + _json_config_append(cfg, 'c', {'c4': 4}) + assert cfg == {'c': {'c3': 3, 'c4': 4}} + _json_config_append(cfg, 'c', {'c4': '44', 'c5': '55'}) + assert cfg == {'c': {'c3': 3, 'c4': '44', 'c5': '55'}} + + def test_load_json_configs_set(self, tmpdir): + cfgfile = tmpdir / 'test.json' + cfg = { + 'str': 'def', + 'int': 1, + 'list': ['a', 1], + 'dict': {'b': 2, 'c': {'d': None}}, + } + cfgfile.write(json.dumps(cfg)) + + cset, cappend = load_json_configs([str(cfgfile)]) + assert cset == cfg + assert cappend == {} + + def test_load_json_configs_append(self, tmpdir): + cfgfile = tmpdir / 'test.json' + cfg = { + '_mode': 'append', + 'list': ['a', 1], + 'dict': {'b': 2, 'c': {'d': None}}, + } + cfgfile.write(json.dumps(cfg)) + + cset, cappend = load_json_configs([str(cfgfile)]) + assert cset == {} + assert cappend == { + 'list': ['a', 1], + 'dict': {'b': 2, 'c': {'d': None}}, + } + + def test_load_json_config_dir(self, tmpdir): + cfgfile1 = tmpdir / 'test1.json' + cfg1 = { + '_mode': 'set', + 'str': 'def', + 'int': 1, + 'list': ['a', 1], + 'dict': {'b': 2, 'c': {'d': None}}, + } + cfgfile1.write(json.dumps(cfg1)) + + cfgfile2 = tmpdir / 'test2.json' + cfg2 = { + '_mode': 'append', + 'list': ['a', 1], + 'dict': {'b': 2, 'c': {'d': None}}, + } + cfgfile2.write(json.dumps(cfg2)) + + cfgfile3 = tmpdir / 'test3.json' + cfg3 = { + '_mode': 'append', + 'list': [None, {'1': 123}], + 'dict': {'e': [1, 2], 'c': {'f': 54321}}, + } + cfgfile3.write(json.dumps(cfg3)) + + cfgfile4 = tmpdir / 'test4.json' + cfg4 = { + 'int': None, + 'dict': {} + } + cfgfile4.write(json.dumps(cfg4)) + + cset, cappend = load_json_config_dir(str(tmpdir)) + assert cset == { + 'str': 'def', + 'list': ['a', 1], + 'dict': {}, + } + assert cappend == { + 'list': ['a', 1, None, {'1': 123}], + 'dict': {'b': 2, 'c': {'f': 54321}, 'e': [1, 2]}, + } diff --git a/components/tools/OmeroWeb/omeroweb/api/api_settings.py b/components/tools/OmeroWeb/omeroweb/api/api_settings.py index e458198de5e..b491da0d1ea 100644 --- a/components/tools/OmeroWeb/omeroweb/api/api_settings.py +++ b/components/tools/OmeroWeb/omeroweb/api/api_settings.py @@ -20,8 +20,12 @@ """Settings for the OMERO JSON api app.""" import sys -from omeroweb.settings import process_custom_settings, report_settings, \ - str_slash +from omeroweb.settings import ( + int_plain, + process_custom_settings, + report_settings, + str_slash, +) # load settings API_SETTINGS_MAPPING = { @@ -29,12 +33,12 @@ "omero.web.api.limit": ["API_LIMIT", 200, - int, + int_plain, "Default number of items returned from json api."], "omero.web.api.max_limit": ["API_MAX_LIMIT", 500, - int, + int_plain, "Maximum number of items returned from json api."], "omero.web.api.absolute_url": ["API_ABSOLUTE_URL", diff --git a/components/tools/OmeroWeb/omeroweb/settings.py b/components/tools/OmeroWeb/omeroweb/settings.py index a54aeb338ac..7068be0b147 100644 --- a/components/tools/OmeroWeb/omeroweb/settings.py +++ b/components/tools/OmeroWeb/omeroweb/settings.py @@ -36,6 +36,7 @@ import omero.clients import tempfile import re +import inspect import json import random import string @@ -166,6 +167,7 @@ } +JSON_CONFIG_DIR = os.getenv('OMERO_WEB_CONFIG_DIR') CONFIG_XML = os.path.join(OMERO_HOME, 'etc', 'grid', 'config.xml') count = 10 event = get_event("websettings") @@ -193,6 +195,12 @@ exctype, value = sys.exc_info()[:2] raise exctype(value) +CUSTOM_SETTINGS_JSON_SET = dict() +CUSTOM_SETTINGS_JSON_APPEND = dict() +if JSON_CONFIG_DIR: + CUSTOM_SETTINGS_JSON_SET, CUSTOM_SETTINGS_JSON_APPEND = \ + omero.config.load_json_config_dir(JSON_CONFIG_DIR) + del event del count del get_event @@ -212,7 +220,9 @@ 'django.contrib.sessions.backends.cached_db') -def parse_boolean(s): +def parse_boolean(s, src=None): + if src == 'json': + return s s = s.strip().lower() if s in ('true', '1', 't'): return True @@ -220,10 +230,10 @@ def parse_boolean(s): def parse_paths(s): - return [os.path.normpath(path) for path in json.loads(s)] + return [os.path.normpath(path) for path in parse_json(s)] -def check_server_type(s): +def check_server_type(s, src=None): if s not in ALL_SERVER_TYPES: raise ValueError( "Unknown server type: %s. Valid values are: %s" @@ -231,7 +241,7 @@ def check_server_type(s): return s -def check_session_engine(s): +def check_session_engine(s, src=None): if s not in SESSION_ENGINE_VALUES: raise ValueError( "Unknown session engine: %s. Valid values are: %s" @@ -239,11 +249,17 @@ def check_session_engine(s): return s -def identity(x): +def identity(x, src=None): return x -def str_slash(s): +def parse_json(j, src=None): + if src == 'json': + return j + return json.loads(j) + + +def str_slash(s, src=None): if s is not None: s = str(s) if s and not s.endswith("/"): @@ -251,17 +267,33 @@ def str_slash(s): return s +def str_plain(s, src=None): + return str(s) + + +def int_plain(i, src=None): + return int(i) + + +def normpath_plain(p, src=None): + return os.path.normpath(p) + + +def str_compile(s, src=None): + return re.compile(s) + + class LeaveUnset(Exception): pass -def leave_none_unset(s): +def leave_none_unset(s, src=None): if s is None: raise LeaveUnset() return s -def leave_none_unset_int(s): +def leave_none_unset_int(s, src=None): s = leave_none_unset(s) if s is not None: return int(s) @@ -272,7 +304,7 @@ def leave_none_unset_int(s): # DO NOT EDIT! INTERNAL_SETTINGS_MAPPING = { "omero.qa.feedback": - ["FEEDBACK_URL", "http://qa.openmicroscopy.org.uk", str, None], + ["FEEDBACK_URL", "http://qa.openmicroscopy.org.uk", str_plain, None], "omero.web.upgrades.url": ["UPGRADES_URL", None, leave_none_unset, None], "omero.web.check_version": @@ -281,7 +313,7 @@ def leave_none_unset_int(s): # Allowed hosts: # https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts "omero.web.allowed_hosts": - ["ALLOWED_HOSTS", '["*"]', json.loads, None], + ["ALLOWED_HOSTS", '["*"]', parse_json, None], # Do not show WARNING (1_8.W001): The standalone TEMPLATE_* settings # were deprecated in Django 1.8 and the TEMPLATES dictionary takes @@ -289,7 +321,7 @@ def leave_none_unset_int(s): # into your default TEMPLATES dict: # TEMPLATE_DIRS, TEMPLATE_CONTEXT_PROCESSORS. "omero.web.system_checks": - ["SILENCED_SYSTEM_CHECKS", '["1_8.W001"]', json.loads, None], + ["SILENCED_SYSTEM_CHECKS", '["1_8.W001"]', parse_json, None], # Internal email notification for omero.web.admins, # loaded from config.xml directly @@ -323,7 +355,7 @@ def leave_none_unset_int(s): "omero.web.admins.email_subject_prefix": ["EMAIL_SUBJECT_PREFIX", "[OMERO.web - admin notification]", - str, + str_plain, "Subject-line prefix for email messages"], "omero.mail.smtp.starttls.enable": ["EMAIL_USE_TLS", @@ -351,7 +383,7 @@ def leave_none_unset_int(s): "omero.web.admins": ["ADMINS", '[]', - json.loads, + parse_json, ("A list of people who get code error notifications whenever the " "application identifies a broken link or raises an unhandled " "exception that results in an internal server error. This gives " @@ -369,12 +401,17 @@ def leave_none_unset_int(s): "omero.web.application_server.host": ["APPLICATION_SERVER_HOST", "127.0.0.1", - str, + str_plain, "Upstream application host"], "omero.web.application_server.port": - ["APPLICATION_SERVER_PORT", 4080, int, "Upstream application port"], + ["APPLICATION_SERVER_PORT", + 4080, + int_plain, + "Upstream application port"], "omero.web.application_server.max_requests": - ["APPLICATION_SERVER_MAX_REQUESTS", 0, int, + ["APPLICATION_SERVER_MAX_REQUESTS", + 0, + int_plain, ("The maximum number of requests a worker will process before " "restarting.")], "omero.web.middleware": @@ -393,7 +430,7 @@ def leave_none_unset_int(s): '{"index": 6, ' '"class": "django.middleware.clickjacking.XFrameOptionsMiddleware"}' ']'), - json.loads, + parse_json, ('Warning: Only system administrators should use this feature. ' 'List of Django middleware classes in the form ' '[{"class": "class.name", "index": FLOAT}]. ' @@ -423,7 +460,7 @@ def leave_none_unset_int(s): "omero.web.static_root": ["STATIC_ROOT", os.path.join(os.path.dirname(__file__), 'static').replace('\\', '/'), - os.path.normpath, + normpath_plain, ("The absolute path to the directory where collectstatic will" " collect static files for deployment. If the staticfiles contrib" " app is enabled (default) the collectstatic management command" @@ -448,7 +485,7 @@ def leave_none_unset_int(s): ["CACHES", ('{"default": {"BACKEND":' ' "django.core.cache.backends.dummy.DummyCache"}}'), - json.loads, + parse_json, ("OMERO.web offers alternative session backends to automatically" " delete stale data using the cache session store backend, see " ":djangodoc:`Django cached session documentation `_" @@ -658,26 +698,26 @@ def leave_none_unset_int(s): "omero.web.login_view": ["LOGIN_VIEW", "weblogin", - str, + str_plain, ("The Django view name used for login. Use this to provide an " "alternative login workflow.")], "omero.web.login_incorrect_credentials_text": ["LOGIN_INCORRECT_CREDENTIALS_TEXT", "Connection not available, please check your user name and password.", - str, + str_plain, ("The error message shown to users who enter an incorrect username " "or password.")], "omero.web.top_logo": ["TOP_LOGO", "", - str, + str_plain, ("Customize the webclient top bar logo. The recommended image height " "is 23 pixels and it must be hosted outside of OMERO.web.")], "omero.web.top_logo_link": ["TOP_LOGO_LINK", "", - str, + str_plain, ("The target location of the webclient top logo, default unlinked.")], "omero.web.user_dropdown": @@ -704,7 +744,7 @@ def leave_none_unset_int(s): "omero.web.staticfile_dirs": ["STATICFILES_DIRS", '[]', - json.loads, + parse_json, ("Defines the additional locations the staticfiles app will traverse" " if the FileSystemFinder finder is enabled, e.g. if you use the" " collectstatic or findstatic management command or use the static" @@ -712,7 +752,7 @@ def leave_none_unset_int(s): "omero.web.template_dirs": ["TEMPLATE_DIRS", '[]', - json.loads, + parse_json, ("List of locations of the template source files, in search order. " "Note that these paths should use Unix-style forward slashes.")], "omero.web.index_template": @@ -730,7 +770,7 @@ def leave_none_unset_int(s): "omero.web.login_redirect": ["LOGIN_REDIRECT", '{}', - json.loads, + parse_json, ("Redirect to the given location after logging in. It only supports " "arguments for :djangodoc:`Django reverse function" " `. " @@ -746,27 +786,27 @@ def leave_none_unset_int(s): "omero.web.login.client_downloads_base": ["CLIENT_DOWNLOAD_GITHUB_REPO", 'ome/omero-insight', - str, + str_plain, ("GitHub repository containing the Desktop client downloads")], "omero.web.apps": ["ADDITIONAL_APPS", '[]', - json.loads, + parse_json, ("Add additional Django applications. For example, see" " :doc:`/developers/Web/CreateApp`")], "omero.web.databases": - ["DATABASES", '{}', json.loads, None], + ["DATABASES", '{}', parse_json, None], "omero.web.page_size": ["PAGE", 200, - int, + int_plain, ("Number of images displayed within a dataset or 'orphaned'" " container to prevent from loading them all at once.")], "omero.web.thumbnails_batch": ["THUMBNAILS_BATCH", 50, - int, + int_plain, ("Number of thumbnails retrieved to prevent from loading them" " all at once. Make sure the size is not too big, otherwise" " you may exceed limit request line, see" @@ -781,7 +821,7 @@ def leave_none_unset_int(s): '["Help", "https://help.openmicroscopy.org/",' '{"title":"Open OMERO user guide in a new tab", "target":"new"}]' ']'), - json.loads, + parse_json, ("Add links to the top header: links are ``['Link Text', " "'link|lookup_view', options]``, where the url is reverse('link'), " "simply 'link' (for external urls) or lookup_view is a detailed " @@ -803,7 +843,7 @@ def leave_none_unset_int(s): '{"name": "rating", "label": "Ratings", "index": 6},' '{"name": "other", "label": "Others", "index": 7}' ']'), - json.loads, + parse_json, ("Manage Metadata pane accordion. This functionality is limited to" " the existing sections.")], "omero.web.ui.right_plugins": @@ -815,7 +855,7 @@ def leave_none_unset_int(s): # "image_roi_tab"],' '["Preview", "webclient/data/includes/right_plugin.preview.js.html"' ', "preview_tab"]]'), - json.loads, + parse_json, ("Add plugins to the right-hand panel. " "Plugins are ``['Label', 'include.js', 'div_id']``. " "The javascript loads data into ``$('#div_id')``.")], @@ -826,7 +866,7 @@ def leave_none_unset_int(s): # "webtest/webclient_plugins/center_plugin.splitview.js.html", # "split_view_panel"],' ']'), - json.loads, + parse_json, ("Add plugins to the center panels. Plugins are " "``['Channel overlay'," " 'webtest/webclient_plugins/center_plugin.overlay.js.html'," @@ -837,7 +877,7 @@ def leave_none_unset_int(s): "omero.web.cors_origin_whitelist": ["CORS_ORIGIN_WHITELIST", '[]', - json.loads, + parse_json, ("A list of origin hostnames that are authorized to make cross-site " "HTTP requests. " "Used by the django-cors-headers app as described at " @@ -852,14 +892,14 @@ def leave_none_unset_int(s): "omero.web.x_frame_options": ["X_FRAME_OPTIONS", "SAMEORIGIN", - str, + str_plain, "Whether to allow OMERO.web to be loaded in a frame." ], "omero.web.django_additional_settings": ["DJANGO_ADDITIONAL_SETTINGS", "[]", - json.loads, + parse_json, ("Additional Django settings as list of key-value tuples. " "Use this to set or override Django settings that aren't managed by " "OMERO.web. E.g. ``[\"CUSTOM_KEY\", \"CUSTOM_VALUE\"]``")], @@ -900,7 +940,7 @@ def leave_none_unset_int(s): "omero.web.email_subject_prefix": ["EMAIL_SUBJECT_PREFIX", "[OMERO.web]", - str, + str_plain, ("Default email subject is no longer configurable.")], "omero.web.email_use_tls": ["EMAIL_USE_TLS", @@ -930,7 +970,7 @@ def leave_none_unset_int(s): del CUSTOM_HOST -def check_worker_class(c): +def check_worker_class(c, src=None): if c == "gevent": try: import gevent # NOQA @@ -940,7 +980,7 @@ def check_worker_class(c): return str(c) -def check_threading(t): +def check_threading(t, src=None): if t > 1: try: import concurrent.futures # NOQA @@ -964,7 +1004,7 @@ def check_threading(t): "omero.web.wsgi_worker_connections": ["WSGI_WORKER_CONNECTIONS", 1000, - int, + int_plain, ("(ASYNC WORKERS only) The maximum number of simultaneous clients. " "Check Gunicorn Documentation https://docs.gunicorn.org" "/en/stable/settings.html#worker-connections")], @@ -993,6 +1033,94 @@ def map_deprecated_settings(settings): return m +def _apply_mapping(value, src, mapping, key): + """ + Applies the mapping function for an OMERO.web setting. + This attempts to handle both new mapping functions (value, src) and old + (value). + Old mapping functions only work with config.xml settings, they do not + support new JSON settings. + + :param value: The raw config value + :param src str: The source of the value + :param mapping function: The mapping function + :param key str: The key name, used to log a deprecation message if an old + mapping function is found + :return (value, unset): + value: The mapped value + unset: If True the property should be left unset + """ + try: + argspec = inspect.getargspec(mapping) + isold = len(argspec.args) < 2 + except TypeError: + # E.g. int(), str() + isold = True + if isold: + logger.warn( + 'Setting %s uses a deprecated mapping function %s', + key, mapping.__name__) + if src not in ('default', 'xml'): + raise ValueError( + 'Deprecated mapping function cannot be used with JSON ' + 'configuration for key %s' % key) + try: + return mapping(value), False + except LeaveUnset: + return None, True + try: + return mapping(value, src), False + except LeaveUnset: + return None, True + + +def lookup_web_config(key, default_value=None, mapping=identity): + """ + Lookup an omero config property as seen by OMERO.web, taking into account + the config.xml and and config JSON files. + + :param key str: The omero config property name + :param default_value: The default value of the property + :param mapping func: The mapping function(value, src) to convert the + property to the OMERO.web value + + :return (value, src, unset): + value: the value of the property taking into account the + default_value and mapping parameters + src: whether the property is the 'default', from the 'xml' config, + or from the 'json', config + unset: If True the property should be left unset, e.g. it should + inherit the default Django value + """ + if (key in CUSTOM_SETTINGS_JSON_SET or + key in CUSTOM_SETTINGS_JSON_APPEND): + try: + global_value = CUSTOM_SETTINGS_JSON_SET[key] + src = 'json' + except KeyError: + global_value = default_value + src = 'default' + global_value, unset = _apply_mapping(global_value, src, mapping, key) + if key in CUSTOM_SETTINGS_JSON_APPEND: + src = 'json' + try: + global_value.extend(CUSTOM_SETTINGS_JSON_APPEND[key]) + except AttributeError: + global_value.update(CUSTOM_SETTINGS_JSON_APPEND[key]) + global_value, unset = _apply_mapping( + global_value, src, mapping, key) + else: + try: + src = 'xml' + global_value = CUSTOM_SETTINGS[key] + except KeyError: + src = 'default' + global_value = default_value + global_value, unset = _apply_mapping( + global_value, src, mapping, key) + return global_value, src, unset + + def process_custom_settings( module, settings='CUSTOM_SETTINGS_MAPPINGS', deprecated=None): logging.info('Processing custom settings for module %s' % module.__name__) @@ -1011,27 +1139,24 @@ def process_custom_settings( continue global_name, default_value, mapping, description = values - try: - global_value = CUSTOM_SETTINGS[key] - values.append(False) - except KeyError: - global_value = default_value - values.append(True) + global_value, src, unset = lookup_web_config( + key, default_value, mapping) + values.append(src) - try: - using_default = values[-1] if global_name in deprecated_map: dep_value, dep_key = deprecated_map[global_name] - if using_default: + if src == 'default': logging.warning( 'Setting %s is deprecated, use %s', dep_key, key) - global_value = dep_value + global_value, unset = _apply_mapping( + global_value, src, mapping, dep_key) else: logging.error( '%s and its deprecated key %s are both set, using %s', key, dep_key, key) - setattr(module, global_name, mapping(global_value)) + if not unset: + setattr(module, global_name, global_value) except ValueError, e: raise ValueError( "Invalid %s (%s = %r). %s. %s" % @@ -1040,8 +1165,6 @@ def process_custom_settings( raise ImportError( "ImportError: %s. %s (%s = %r).\n%s" % (e.message, global_name, key, global_value, description)) - except LeaveUnset: - pass process_custom_settings(sys.modules[__name__], 'INTERNAL_SETTINGS_MAPPING') @@ -1060,9 +1183,9 @@ def report_settings(module): custom_settings_mappings = getattr(module, 'CUSTOM_SETTINGS_MAPPINGS', {}) for key in sorted(custom_settings_mappings): values = custom_settings_mappings[key] - global_name, default_value, mapping, description, using_default = \ - values - source = using_default and "default" or key + global_name, default_value, mapping, description, source = values + if source != 'default': + source = '%s:%s' % (source, key) global_value = getattr(module, global_name, None) if global_name.isupper(): logger.debug( @@ -1072,10 +1195,9 @@ def report_settings(module): deprecated_settings = getattr(module, 'DEPRECATED_SETTINGS_MAPPINGS', {}) for key in sorted(deprecated_settings): values = deprecated_settings[key] - global_name, default_value, mapping, description, using_default = \ - values + global_name, default_value, mapping, description, source = values global_value = getattr(module, global_name, None) - if global_name.isupper() and not using_default: + if global_name.isupper() and source != 'default': logger.debug( "%s = %r (deprecated:%s, %s)", global_name, cleanse_setting(global_name, global_value), key, description) diff --git a/components/tools/OmeroWeb/test/unit/test_settings.py b/components/tools/OmeroWeb/test/unit/test_settings.py index 6f90c4a4bc1..675c5541cf0 100644 --- a/components/tools/OmeroWeb/test/unit/test_settings.py +++ b/components/tools/OmeroWeb/test/unit/test_settings.py @@ -23,6 +23,15 @@ """ from connector import Server +import settings +from settings import ( + _apply_mapping, + int_plain, + leave_none_unset, + lookup_web_config, + parse_json, + str_plain, +) # Test model @@ -81,3 +90,80 @@ def test_load_server_list(self): assert str(te) == 'No more instances allowed' Server(host=u'example1.com', port=4064) + + +class TestProperties(object): + + def test_apply_mapping_leaveunset(self): + assert (None, True) == _apply_mapping( + None, 'default', leave_none_unset, 'test.unset') + assert ('x', False) == _apply_mapping( + 'x', 'default', leave_none_unset, 'test.unset') + + def test_apply_mapping_oldmapping(self): + assert (123, False) == _apply_mapping( + '123', 'default', int, 'test.oldmapping') + + def test_apply_mapping_parsejson(self): + assert ({'a': 1}, False) == _apply_mapping( + '{"a": 1}', 'default', parse_json, 'test.parsejson') + assert ({'a': 1}, False) == _apply_mapping( + '{"a": 1}', 'xml', parse_json, 'test.parsejson') + assert ({'a': 1}, False) == _apply_mapping( + {'a': 1}, 'json', parse_json, 'test.parsejson') + + def test_lookup_web_config_default(self, monkeypatch): + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS_JSON_SET', {}) + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS_JSON_APPEND', {}) + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS', {}) + + assert (123, 'default', False) == lookup_web_config( + 'test.missing', 123) + assert ('123', 'default', False) == lookup_web_config( + 'test.missing', 123, str_plain) + + def test_lookup_web_config_jsonset(self, monkeypatch): + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS_JSON_SET', { + 'test.k1': 1, + 'test.k2': [2], + }) + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS_JSON_APPEND', {}) + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS', { + 'test.k1': 'should be ignored', + }) + + assert (1, 'json', False) == lookup_web_config('test.k1') + assert (123, 'default', False) == lookup_web_config( + 'test.missing', 123) + + assert (1, 'json', False) == lookup_web_config( + 'test.k1', '123', parse_json) + assert (123, 'default', False) == lookup_web_config( + 'test.missing', '123', parse_json) + + def test_lookup_web_config_jsonappend(self, monkeypatch): + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS_JSON_SET', { + 'test.k1': [1], + }) + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS_JSON_APPEND', { + 'test.k1': [2, 3], + 'test.k2': [4, 5], + }) + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS', { + 'test.k1': 'should be ignored', + }) + assert ([1, 2, 3], 'json', False) == lookup_web_config( + 'test.k1', '[]', parse_json) + assert ([4, 5], 'json', False) == lookup_web_config( + 'test.k2', '[]', parse_json) + + def test_lookup_web_config_xml(self, monkeypatch): + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS_JSON_SET', {}) + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS_JSON_APPEND', {}) + monkeypatch.setattr(settings, 'CUSTOM_SETTINGS', { + 'test.k1': '1', + }) + + assert ('1', 'xml', False) == lookup_web_config('test.k1', 123) + assert (1, 'xml', False) == lookup_web_config( + 'test.k1', 123, int_plain)