diff --git a/benedict/dicts/__init__.py b/benedict/dicts/__init__.py index 8fefd3c8..4250ebeb 100644 --- a/benedict/dicts/__init__.py +++ b/benedict/dicts/__init__.py @@ -191,9 +191,9 @@ def match(self, pattern, indexes=True): def merge(self, other, *args, **kwargs): """ Merge one or more dict objects into current instance (deepupdate). - Sub-dictionaries will be merged toghether. + Sub-dictionaries will be merged together. If overwrite is False, existing values will not be overwritten. - If concat is True, list values will be concatenated toghether. + If concat is True, list values will be concatenated together. """ _merge(self, other, *args, **kwargs) @@ -209,7 +209,7 @@ def nest( ): """ Nest a list of dicts at the given key and return a new nested list - using the specified keys to establish the correct items hierarchy. + using the specified keys to establish the correct item's hierarchy. """ return _nest(self[key], id_key, parent_id_key, children_key) diff --git a/benedict/dicts/keylist/keylist_dict.py b/benedict/dicts/keylist/keylist_dict.py index 2f82dcea..38f2b07f 100644 --- a/benedict/dicts/keylist/keylist_dict.py +++ b/benedict/dicts/keylist/keylist_dict.py @@ -26,7 +26,10 @@ def __delitem__(self, key): def _delitem_by_keys(self, keys): parent, key, _ = keylist_util.get_item(self, keys) - if type_util.is_dict_or_list(parent): + if type_util.is_wildcard(key): + self[keys[:-1]].clear() + return + elif type_util.is_dict_or_list(parent): del parent[key] return elif type_util.is_tuple(parent): @@ -41,6 +44,10 @@ def __getitem__(self, key): def _getitem_by_keys(self, keys): parent, key, _ = keylist_util.get_item(self, keys) + if type_util.is_list(parent) and type_util.is_wildcard(key): + return parent + if type_util.is_list_of_dicts(parent) and type_util.any_wildcard_in_list(keys): + return [item.get(key) for item in parent] if type_util.is_dict_or_list_or_tuple(parent): return parent[key] raise KeyError(f"Invalid keys: '{keys}'") @@ -61,7 +68,14 @@ def get(self, key, default=None): def _get_by_keys(self, keys, default=None): parent, key, _ = keylist_util.get_item(self, keys) - if type_util.is_dict(parent): + if type_util.is_list(parent) and type_util.is_wildcard(key): + return parent + elif type_util.is_wildcard(keys[-2]): + if type_util.is_list_of_dicts(parent): + return [item.get(key) for item in parent] + elif type_util.is_list_of_list(parent): + return _ + elif type_util.is_dict(parent): return parent.get(key, default) elif type_util.is_list_or_tuple(parent): return parent[key] @@ -76,6 +90,13 @@ def _pop_by_keys(self, keys, *args): parent, key, _ = keylist_util.get_item(self, keys) if type_util.is_dict(parent): return parent.pop(key, *args) + elif type_util.is_list(parent) and type_util.is_wildcard(key): + del self[keys[:-1]] + return parent + elif type_util.is_list_of_dicts(parent) and type_util.any_wildcard_in_list( + keys + ): + return [_item.pop(key) if key in _item else None for _item in parent] elif type_util.is_list(parent): return parent.pop(key) elif type_util.is_tuple(parent): diff --git a/benedict/dicts/keylist/keylist_util.py b/benedict/dicts/keylist/keylist_util.py index 6c7185d4..06d3bce2 100644 --- a/benedict/dicts/keylist/keylist_util.py +++ b/benedict/dicts/keylist/keylist_util.py @@ -1,3 +1,5 @@ +from itertools import chain + from benedict.utils import type_util @@ -7,14 +9,30 @@ def _get_index(key): return None -def _get_item_key_and_value(item, key): +def _get_item_key_and_value(item, index, parent=None): if type_util.is_list_or_tuple(item): - index = _get_index(key) - if index is not None: - return (index, item[index]) + if type_util.is_wildcard(index): + return index, item + elif type_util.is_wildcard(parent): + if type_util.is_list_of_dicts(item) and any( + index in _item.keys() for _item in item + ): + return index, [ + _item.get(index) for _item in item if index in _item.keys() + ] + elif type_util.is_list_of_list(item): + return index, [ + _item.get(index) + for _item in chain.from_iterable(item) + if index in _item.keys() + ] + else: + index = _get_index(index) + if index is not None: + return index, item[index] elif type_util.is_dict(item): - return (key, item[key]) - raise KeyError(f"Invalid key: '{key}'") + return index, item[index] + raise KeyError(f"Invalid key: '{index}'") def _get_or_new_item_value(item, key, subkey): @@ -43,6 +61,10 @@ def _set_item_value(item, key, value): # insert index item += [None] * (index - len(item)) item.insert(index, value) + elif type_util.is_list(item): + for idx, _item in enumerate(value): + if _item is not None: + item[idx].update({key: _item}) else: item[key] = value @@ -57,7 +79,16 @@ def get_items(d, keys): item = d for key in keys: try: - item_key, item_value = _get_item_key_and_value(item, key) + if any(items): + if type_util.is_wildcard(val=key): + parent = items[-1][1] + elif type_util.is_wildcard(items[-1][1]): + parent = items[-1][1] + else: + parent = None + else: + parent = None + item_key, item_value = _get_item_key_and_value(item, key, parent) items.append((item, item_key, item_value)) item = item_value except (IndexError, KeyError): diff --git a/benedict/dicts/keypath/keypath_util.py b/benedict/dicts/keypath/keypath_util.py index 50796d4a..dfcf1045 100644 --- a/benedict/dicts/keypath/keypath_util.py +++ b/benedict/dicts/keypath/keypath_util.py @@ -4,6 +4,7 @@ from benedict.utils import type_util KEY_INDEX_RE = r"(?:\[[\'\"]*(\-?[\d]+)[\'\"]*\]){1}$" +KEY_WILDCARD_RE = r"(?:\[[\'\"]*(\-?[\*]+)[\'\"]*\]){1}$" def check_keys(d, separator): @@ -42,12 +43,17 @@ def _split_key_indexes(key): if "[" in key and key.endswith("]"): keys = [] while True: - matches = re.findall(KEY_INDEX_RE, key) - if matches: + index_matches = re.findall(KEY_INDEX_RE, key) + if index_matches: key = re.sub(KEY_INDEX_RE, "", key) - index = int(matches[0]) + index = int(index_matches[0]) + keys.insert(0, index) + continue + index_matches = re.findall(KEY_WILDCARD_RE, key) + if bool(re.search(KEY_WILDCARD_RE, key)): + key = re.sub(KEY_WILDCARD_RE, "", key) + index = index_matches[0] keys.insert(0, index) - # keys.insert(0, { keylist_util.INDEX_KEY:index }) continue keys.insert(0, key) break diff --git a/benedict/utils/type_util.py b/benedict/utils/type_util.py index 476d66e5..38aca82b 100644 --- a/benedict/utils/type_util.py +++ b/benedict/utils/type_util.py @@ -93,3 +93,19 @@ def is_tuple(val): def is_uuid(val): return is_string(val) and uuid_re.match(val) + + +def is_wildcard(val): + return is_string(val) and val == "*" + + +def is_list_of_dicts(val): + return is_list(val) and all(is_dict(_val) for _val in val) + + +def any_wildcard_in_list(val): + return is_list(val) and any(is_wildcard(_val) for _val in val) + + +def is_list_of_list(val): + return is_list(val) and all(is_list(_val) for _val in val) diff --git a/tests/dicts/keypath/test_keypath_dict_list_wildcard.py b/tests/dicts/keypath/test_keypath_dict_list_wildcard.py new file mode 100644 index 00000000..b7eb2b3e --- /dev/null +++ b/tests/dicts/keypath/test_keypath_dict_list_wildcard.py @@ -0,0 +1,458 @@ +import unittest + +from benedict.dicts import KeypathDict + + +class keypath_dict_list_wildcard_test_case(unittest.TestCase): + def test_correct_wildcard(self): + self.kd = KeypathDict( + { + "a": [ + {"x": 1, "y": 1}, + {"x": 2, "y": 2}, + ], + } + ) + correct_wildcard_path_example = "a[*].x" + self.assertEqual(self.kd[correct_wildcard_path_example], [1, 2]) + + def test_wildcard_contains_with_flat_list(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + self.assertTrue("a[*]" in b) + + def test_wildcard_contains_with_wrong_property(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + self.assertFalse("b[*]" in b) + + def test_wildcard_contains_with_nested_list(self): + d = { + "a": { + "b": [ + { + "c": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "d": [4, 5, 6, [0]], + }, + ], + }, + } + b = KeypathDict(d) + + self.assertTrue("a.b[0].d[*]" in b) + self.assertTrue("a.b[1].d[*]" in b) + self.assertTrue("a.b[*]" in b) + self.assertFalse("a.c[*]" in b) + self.assertFalse("a.b.c[*]" in b) + self.assertFalse("a.b[0].c[*]" in b) + self.assertTrue("a.b[0].d[*][*]" in b) + self.assertTrue("a.b[1].d[*][*]" in b) + + def test_wildcard_delitem_with_flat_list(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + del b["a[*]"] + self.assertEqual(b, {"a": []}) + d1 = { + "a": [1, 2, 3], + } + b1 = KeypathDict(d1) + del b1["a[-1]"] + del b1["a[1]"] + del b1["a[0]"] + self.assertEqual(b, {"a": []}) + self.assertEqual(True, b == b1) + + def test_wildcard_delitem_with_wrong_index(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + with self.assertRaises(KeyError): + del b["b[*]"] + + def test_wildcard_delitem_with_nested_list(self): + d = { + "a": { + "b": [ + { + "c": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "d": [4, 5, 6, [0]], + }, + ], + }, + } + b = KeypathDict(d) + + del b["a.b[0].d[*]"] + self.assertEqual(b["a.b[0].d"], []) + + del b["a.b[0].d"] + self.assertEqual(b["a.b[0]"], {"c": 1}) + + del b["a.b[*]"] + self.assertEqual(b["a.b"], []) + + def test_wildcard_getitem_with_flat_list(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + + self.assertEqual(b["a[*]"], [1, 2, 3]) + + def test_wildcard_getitem_with_wrong_index(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + with self.assertRaises(KeyError): + self.assertEqual(b["b[*]"], 1) + + def test_wildcard_getitem_with_nested_list(self): + d = { + "a": { + "b": [ + { + "c": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "d": [4, 5, 6, [0]], + }, + { + "c": 3, + "d": [7, 8, 9, [0]], + }, + ], + }, + } + b = KeypathDict(d) + + self.assertEqual(b["a.b[0].d[*]"], [1, 2, 3, [0]]) + self.assertEqual(b["a.b[1].d[*]"], [4, 5, 6, [0]]) + self.assertEqual(b["a.b[2].d[*]"], [7, 8, 9, [0]]) + + self.assertEqual( + b["a.b[*]"], + [ + { + "c": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "d": [4, 5, 6, [0]], + }, + { + "c": 3, + "d": [7, 8, 9, [0]], + }, + ], + ) + self.assertEqual(b["a.b[2].d[3][*]"], [0]) + + def test_wildcard_getitem_github_issue_feature_request(self): + d = { + "products": [ + { + "categories": [ + { + "name": "OK 1", + }, + { + "name": "OK 2", + }, + ], + }, + ], + } + b = KeypathDict(d) + self.assertEqual( + b['products[""0""].categories[*]'], + [ + { + "name": "OK 1", + }, + { + "name": "OK 2", + }, + ], + ) + + def test_wildcard_get_with_flat_list(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + + self.assertEqual(b.get("a[*]"), [1, 2, 3]) + + def test_wildcard_get_with_wrong_index(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + self.assertEqual(b.get("b[*]", 1), 1) + + def test_wildcard_get_with_nested_list(self): + d = { + "a": { + "b": [ + { + "c": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "d": [4, 5, 6, [0]], + }, + { + "c": 3, + "d": [7, 8, 9, [0]], + }, + ], + }, + } + b = KeypathDict(d) + + self.assertEqual(b.get("a.b[0].d[*]"), [1, 2, 3, [0]]) + self.assertEqual(b.get("a.b[1].d[*]"), [4, 5, 6, [0]]) + self.assertEqual(b.get("a.b[2].d[*]"), [7, 8, 9, [0]]) + + self.assertEqual( + b["a.b[*]"], + [ + { + "c": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "d": [4, 5, 6, [0]], + }, + { + "c": 3, + "d": [7, 8, 9, [0]], + }, + ], + ) + self.assertEqual(b.get("a.b[2].d[3][*]"), [0]) + + def test_wildcard_list_indexes_with_quotes(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + self.assertEqual(b.get("a['*']"), [1, 2, 3]) + self.assertEqual(b.get('a["*"]'), [1, 2, 3]) + + def test_wildcard_pop_with_flat_list(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + + self.assertEqual(b.pop("a[*]"), [1, 2, 3]) + self.assertEqual(b, {}) + + def test_wildcard_pop_with_flat_list_and_default(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + + with self.assertRaises(KeyError): + b.pop("b[*]") + self.assertEqual(b.pop("b[*]", 4), 4) + + def test_wildcard_pop_with_wrong_index(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + with self.assertRaises(KeyError): + b.pop("b[*]") + + def test_wildcard_pop_with_wrong_index_and_default(self): + d = { + "a": [1, 2, 3], + } + b = KeypathDict(d) + self.assertEqual(b.pop("b[*]", 6), 6) + + def test_wildcard_pop_with_nested_list(self): + d = { + "a": { + "b": [ + { + "c": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "d": [4, 5, 6, [0]], + }, + ], + }, + } + b = KeypathDict(d) + + self.assertEqual(b.pop("a.b[0].d[-1][*]"), [0]) + self.assertEqual(b["a.b[0].d"], [1, 2, 3]) + + self.assertEqual(b.pop("a.b[0].d[*]"), [1, 2, 3]) + self.assertEqual(b["a.b[0]"], {"c": 1}) + + self.assertEqual(b.pop("a.b[0]"), {"c": 1}) + self.assertEqual( + b["a.b[0]"], + { + "c": 2, + "d": [4, 5, 6, [0]], + }, + ) + + def test_wildcard_asterix_as_key(self): + d = { + "*": { + "b": [ + { + "*": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "*": [4, 5, 6, [0]], + }, + ], + }, + } + b = KeypathDict(d) + self.assertTrue( + b["*"], + { + "b": [ + { + "*": 1, + "d": [1, 2, 3, [0]], + }, + { + "c": 2, + "*": [4, 5, 6, [0]], + }, + ], + }, + ) + self.assertTrue(b["*.b[0].*"], 1) + self.assertTrue(b.get("*.b[0].*", 2), 1) + + # wrong index => default value should be used + self.assertTrue(b.get("*.b[2].*", 2), 2) + self.assertTrue(b["*.b[1][*]"], [4, 5, 6, [0]]) + + with self.assertRaises(KeyError): + self.assertFalse(b["*.a"], None) + with self.assertRaises(KeyError): + self.assertFalse(b["*.*"], None) + + self.assertEqual(b.get("*.*", "not-existing"), "not-existing") + + def test_complex_wildcard_usage(self): + d = { + "a": { + "b": [ + { + "c": [ + {"x": 10, "y": 20}, + {"x": 11, "y": 21}, + {"x": 12, "y": 22}, + ], + }, + { + "c": [ + {"x": 20, "y": 30}, + {"x": 21, "y": 31}, + {"x": 22, "y": 32}, + ], + }, + ], + }, + } + b = KeypathDict(d) + self.assertEqual( + b.get("a.b[0]"), + { + "c": [ + {"x": 10, "y": 20}, + {"x": 11, "y": 21}, + {"x": 12, "y": 22}, + ], + }, + ) + self.assertEqual( + b.get("a.b[*]"), + [ + { + "c": [ + {"x": 10, "y": 20}, + {"x": 11, "y": 21}, + {"x": 12, "y": 22}, + ], + }, + { + "c": [ + {"x": 20, "y": 30}, + {"x": 21, "y": 31}, + {"x": 22, "y": 32}, + ], + }, + ], + ) + self.assertEqual( + b.get("a.b[0].c"), + [ + {"x": 10, "y": 20}, + {"x": 11, "y": 21}, + {"x": 12, "y": 22}, + ], + ) + self.assertEqual( + b.get("a.b[1].c"), + [ + {"x": 20, "y": 30}, + {"x": 21, "y": 31}, + {"x": 22, "y": 32}, + ], + ) + self.assertEqual(b.get("a.b[1].c[1]"), {"x": 21, "y": 31}) + self.assertEqual( + b.get("a.b[*].c"), + [ + [{"x": 10, "y": 20}, {"x": 11, "y": 21}, {"x": 12, "y": 22}], + [{"x": 20, "y": 30}, {"x": 21, "y": 31}, {"x": 22, "y": 32}], + ], + ) + self.assertEqual( + b.get("a.b[*].c[*].x"), + [10, 11, 12, 20, 21, 22], + ) + self.assertEqual( + b.get("a.b[*].c[*].x[-1]"), + 22, + ) diff --git a/tests/dicts/keypath/test_keypath_util.py b/tests/dicts/keypath/test_keypath_util.py index cd6c2654..3f72e4e9 100644 --- a/tests/dicts/keypath/test_keypath_util.py +++ b/tests/dicts/keypath/test_keypath_util.py @@ -27,7 +27,6 @@ def test_split_key_indexes_with_valid_indexes(self): def test_split_key_indexes_with_invalid_indexes(self): f = keypath_util._split_key_indexes self.assertEqual(f("item[]"), ["item[]"]) - self.assertEqual(f("item[*]"), ["item[*]"]) self.assertEqual(f("item[0:2]"), ["item[0:2]"]) self.assertEqual(f("item[:1]"), ["item[:1]"]) self.assertEqual(f("item[::1]"), ["item[::1]"]) diff --git a/tests/dicts/test_benedict_wildcard.py b/tests/dicts/test_benedict_wildcard.py new file mode 100644 index 00000000..89a8568a --- /dev/null +++ b/tests/dicts/test_benedict_wildcard.py @@ -0,0 +1,74 @@ +import unittest + +from benedict import benedict + + +class keypath_dict_list_wildcard_test_case(unittest.TestCase): + def test_rename_wildcard(self): + d = { + "a": [ + {"x": 1, "y": 1}, + {"x": 2, "y": 2}, + ], + } + d = benedict(d) + b = benedict(d.clone()) + b.rename("a[0].x", "a[0].m") + b.rename("a[1].x", "a[1].m") + + result = { + "a": [ + { + "m": 1, + "y": 1, + }, + { + "m": 2, + "y": 2, + }, + ] + } + + self.assertEqual(b, result) + b = benedict(d.clone()) + b.rename("a[*].x", "a[*].m") + self.assertEqual(b, result) + + def test_swap_wildcard(self): + d = { + "a": [ + {"x": 1, "y": 1}, + {"x": 2, "y": 2}, + ], + "x": [ + {"a": 10, "b": 10}, + {"a": 11, "b": 11}, + ], + } + b = benedict(d) + b.swap("a[*].x", "x[*].a") + result = { + "a": [ + {"x": 10, "y": 1}, + {"x": 11, "y": 2}, + ], + "x": [ + {"a": 1, "b": 10}, + {"a": 2, "b": 11}, + ], + } + self.assertEqual(b, result) + + def test_swap_wildcard_whole_list(self): + d = { + "a": [ + {"x": 1, "y": 1}, + {"x": 2, "y": 2}, + ], + "x": [ + {"a": 10, "b": 10}, + {"a": 11, "b": 11}, + ], + } + b = benedict(d) + b.get("a[1]")