Alternative JSON configuration system for OMERO.web

components/tools/OmeroPy/src/omero/config.py

@@ -30,6 +30,11 @@
 )
 from omero_ext import portalocker
 import json
+try:
+    import yaml
+    YAML_ENABLED = True
+except ImportError:
+    YAML_ENABLED = False
 
 
 class Environment(object):
@@ -439,3 +444,110 @@ def __delitem__(self, key):
                 to_remove.append(p)
         for x in to_remove:
             props.remove(x)
+
+
+def load_json_config_dir(configdir):
+    """
+    Load a directory of .json, .yaml or .yml configuration files.
+    """
+    files = (f for f in os.listdir(configdir) if
+             not f.startswith('.') and
+             os.path.splitext(f)[1].lower() in ('.json', '.yaml', '.yml'))
+    cset, cappend = load_json_configs(
+        os.path.join(configdir, f) for f in sorted(files))
+    return cset, cappend
+
+
+def load_json_configs(configfiles):
+    """
+    Load a set of configuration files in the given order, and return two
+    dictionaries:
+    - values to be set
+    - values to be appended
+
+    The caller should combine these two dictionaries appropriately.
+    Typically properties to be `set` will be processed first to allow existing
+    defaults to be unset if necessary, followed by processing of properties to
+    `append`.
+
+    Each file is a single JSON or YAML dictionary (set of key-value pairs).
+    A special key `_mode` can take the value `"set"` (default) or `"append"`
+    which determines whether the properties will be added to the `set` or
+    `append` config dictionary.
+
+    In `set` mode duplicate properties will overwrite existing ones, this is
+    equivalent to `omero config set` with the following rules:
+    - a null value will unset the configuration key
+    - otherwise the configuration key will be set to the value
+
+    In `append` mode the value of each field must be a list or dictionary, it
+    is not possible to append a scalar.
+    - If the value is a list all values in the list will be appended to the
+      current property value
+    - If the value is a dictionary it will be merged non-recursively to the
+    all values in the list will be appended to the
+      current property value
+
+    :param configfiles [str]: List of JSON or YAML (if module is available)
+    filepaths to load
+    :return (dict, dict): Tuple of config-set and config-append properties
+    """
+    cset = {}
+    cappend = {}
+
+    for f in configfiles:
+        with open(f) as fh:
+            try:
+                if f.lower().endswith('.yaml') or f.lower().endswith('.yml'):
+                    if not YAML_ENABLED:
+                        raise Exception(
+                            'PyYAML module required to load {}'.format(f))
+                    d = yaml.load(fh)
+                else:
+                    d = json.load(fh)
+            except Exception as e:
+                raise Exception('Failed to load file {}: {}'.format(f, e))
+            mode = d.pop('_mode', 'set')
+            for k, v in d.items():
+                if mode == 'set':
+                    _json_config_set(cset, k, v)
+                elif mode == 'append':
+                    _json_config_append(cappend, k, v)
+                else:
+                    raise Exception(
+                        'Invalid configuration mode: {}'.format(mode))
+
+    return cset, cappend
+
+
+def _json_config_set(config, k, v):
+    if v is None:
+        config.pop(k, None)
+    else:
+        config[k] = v
+
+
+def _json_config_append(config, k, v):
+    if isinstance(v, list):
+        if k not in config:
+            config[k] = v
+        else:
+            try:
+                config[k].extend(v)
+            except AttributeError:
+                raise Exception(
+                    'Incompatible types for append key {}: {} {}'.format(
+                        k, config[k], v))
+    elif isinstance(v, dict):
+        if k not in config:
+            config[k] = v
+        else:
+            try:
+                config[k].update(v)
+            except AttributeError:
+                raise Exception(
+                    'Incompatible types for append key {}: {} {}'.format(
+                        k, config[k], v))
+    else:
+        raise Exception(
+            'Append requires a list or dictionary value: {}: {}'.format(k, v))

components/tools/OmeroPy/test/unit/test_config.py

@@ -12,10 +12,18 @@
 import os
 import errno
 import pytest
-from omero.config import ConfigXml, xml
+from omero.config import (
+    ConfigXml,
+    xml,
+    _json_config_append,
+    _json_config_set,
+    load_json_config_dir,
+    load_json_configs,
+)
 from omero.util.temp_files import create_path
 from omero_ext import portalocker
 
+import json
 from xml.etree.ElementTree import XML, Element, SubElement, tostring
 
 
@@ -355,3 +363,126 @@ def testCannotRead(self):
         with pytest.raises(IOError) as excinfo:
             ConfigXml(str(p)).close()
         assert excinfo.value.errno == errno.EACCES
+
+
+class TestJsonConfig(object):
+
+    def test_json_config_set(self):
+        cfg = {}
+        _json_config_set(cfg, 'a', 'a1')
+        assert cfg == {'a': 'a1'}
+        _json_config_set(cfg, 'b', 'b2')
+        assert cfg == {'a': 'a1', 'b': 'b2'}
+        _json_config_set(cfg, 'a', 'a11')
+        assert cfg == {'a': 'a11', 'b': 'b2'}
+
+    def test_json_config_append_invalid(self):
+        cfg = {
+            'a': 1,
+            'b': [2],
+            'c': {'c3': 3},
+        }
+        with pytest.raises(Exception):
+            _json_config_append(cfg, 'a', 'not-list-or-dict')
+        with pytest.raises(Exception):
+            _json_config_append(cfg, 'b', 'not-list-or-dict')
+        with pytest.raises(Exception):
+            _json_config_append(cfg, 'c', 'not-list-or-dict')
+
+        with pytest.raises(Exception):
+            _json_config_append(cfg, 'b', {'not': 'list'})
+        with pytest.raises(Exception):
+            _json_config_append(cfg, 'c', ['not', 'dict'])
+
+    def test_json_config_append_list(self):
+        cfg = {}
+        _json_config_append(cfg, 'b', [2])
+        assert cfg == {'b': [2]}
+        _json_config_append(cfg, 'b', [3])
+        assert cfg == {'b': [2, 3]}
+        _json_config_append(cfg, 'b', [4, 5])
+        assert cfg == {'b': [2, 3, 4, 5]}
+
+    def test_json_config_append_dict(self):
+        cfg = {}
+        _json_config_append(cfg, 'c', {'c3': 3})
+        assert cfg == {'c': {'c3': 3}}
+        _json_config_append(cfg, 'c', {'c4': 4})
+        assert cfg == {'c': {'c3': 3, 'c4': 4}}
+        _json_config_append(cfg, 'c', {'c4': '44', 'c5': '55'})
+        assert cfg == {'c': {'c3': 3, 'c4': '44', 'c5': '55'}}
+
+    def test_load_json_configs_set(self, tmpdir):
+        cfgfile = tmpdir / 'test.json'
+        cfg = {
+            'str': 'def',
+            'int': 1,
+            'list': ['a', 1],
+            'dict': {'b': 2, 'c': {'d': None}},
+        }
+        cfgfile.write(json.dumps(cfg))
+
+        cset, cappend = load_json_configs([str(cfgfile)])
+        assert cset == cfg
+        assert cappend == {}
+
+    def test_load_json_configs_append(self, tmpdir):
+        cfgfile = tmpdir / 'test.json'
+        cfg = {
+            '_mode': 'append',
+            'list': ['a', 1],
+            'dict': {'b': 2, 'c': {'d': None}},
+        }
+        cfgfile.write(json.dumps(cfg))
+
+        cset, cappend = load_json_configs([str(cfgfile)])
+        assert cset == {}
+        assert cappend == {
+            'list': ['a', 1],
+            'dict': {'b': 2, 'c': {'d': None}},
+        }
+
+    def test_load_json_config_dir(self, tmpdir):
+        cfgfile1 = tmpdir / 'test1.json'
+        cfg1 = {
+            '_mode': 'set',
+            'str': 'def',
+            'int': 1,
+            'list': ['a', 1],
+            'dict': {'b': 2, 'c': {'d': None}},
+        }
+        cfgfile1.write(json.dumps(cfg1))
+
+        cfgfile2 = tmpdir / 'test2.json'
+        cfg2 = {
+            '_mode': 'append',
+            'list': ['a', 1],
+            'dict': {'b': 2, 'c': {'d': None}},
+        }
+        cfgfile2.write(json.dumps(cfg2))
+
+        cfgfile3 = tmpdir / 'test3.json'
+        cfg3 = {
+            '_mode': 'append',
+            'list': [None, {'1': 123}],
+            'dict': {'e': [1, 2], 'c': {'f': 54321}},
+        }
+        cfgfile3.write(json.dumps(cfg3))
+
+        cfgfile4 = tmpdir / 'test4.json'
+        cfg4 = {
+            'int': None,
+            'dict': {}
+        }
+        cfgfile4.write(json.dumps(cfg4))
+
+        cset, cappend = load_json_config_dir(str(tmpdir))
+        assert cset == {
+            'str': 'def',
+            'list': ['a', 1],
+            'dict': {},
+        }
+        assert cappend == {
+            'list': ['a', 1, None, {'1': 123}],
+            'dict': {'b': 2, 'c': {'f': 54321}, 'e': [1, 2]},
+        }

components/tools/OmeroWeb/omeroweb/api/api_settings.py

@@ -20,21 +20,25 @@
 """Settings for the OMERO JSON api app."""
 
 import sys
-from omeroweb.settings import process_custom_settings, report_settings, \
-    str_slash
+from omeroweb.settings import (
+    int_plain,
+    process_custom_settings,
+    report_settings,
+    str_slash,
+)
 
 # load settings
 API_SETTINGS_MAPPING = {
 
     "omero.web.api.limit":
         ["API_LIMIT",
          200,
-         int,
+         int_plain,
          "Default number of items returned from json api."],
     "omero.web.api.max_limit":
         ["API_MAX_LIMIT",
          500,
-         int,
+         int_plain,
          "Maximum number of items returned from json api."],
     "omero.web.api.absolute_url":
         ["API_ABSOLUTE_URL",