From 86af6e65a903d8685ed6efac64ae29e1e054c9fc Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Tue, 15 Jun 2021 15:49:26 -0700 Subject: [PATCH 01/16] typo --- changegen/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changegen/generator.py b/changegen/generator.py index b6b915e..927ffd2 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -583,7 +583,7 @@ def generate_changes( new_nodes.extend(nodes) new_ways.extend(ways) _global_node_id_all_ways.extend(chain.from_iterable([w.nds for w in ways])) - if isinstance(wgs84_geom, sg.Polygon): + elif isinstance(wgs84_geom, sg.Polygon): ## If we're taking all features to be newly-created (~modify_only) ## we need to create ways and nodes for that feature. ## IF we're only modifying existing features with features From 256613eeaa306bb4f06705d17473364b01387cec Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Tue, 15 Jun 2021 16:12:58 -0700 Subject: [PATCH 02/16] fix oversight in linestring meta modification (caused in merge conflict) --- changegen/generator.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/changegen/generator.py b/changegen/generator.py index 927ffd2..d730f09 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -573,16 +573,31 @@ def generate_changes( ): raise NotImplementedError("Multi geometries not supported.") if isinstance(wgs84_geom, sg.LineString): - ways, nodes = _generate_ways_and_nodes( - wgs84_geom, - ids, - feat_tags, - intersection_db, - max_nodes_per_way=max_nodes_per_way, - ) - new_nodes.extend(nodes) - new_ways.extend(ways) - _global_node_id_all_ways.extend(chain.from_iterable([w.nds for w in ways])) + ## NOTE that modify_only does not support modifying geometries. + if modify_only: + existing_id = feature.GetFieldAsString(feature.GetFieldIndex("osm_id")) + + new_ways.append( + Way( + id=existing_id, + version=2, + nds=existing_nodes_for_ways[existing_id], + tags=[tag for tag in feat_tags if tag.key != "osm_id"], + ) + ) + else: # not modifying, just creating + ways, nodes = _generate_ways_and_nodes( + wgs84_geom, + ids, + feat_tags, + intersection_db, + max_nodes_per_way=max_nodes_per_way, + ) + new_nodes.extend(nodes) + new_ways.extend(ways) + _global_node_id_all_ways.extend( + chain.from_iterable([w.nds for w in ways]) + ) elif isinstance(wgs84_geom, sg.Polygon): ## If we're taking all features to be newly-created (~modify_only) ## we need to create ways and nodes for that feature. From 2c24c5de61dda94151aabdf93420675ae825a9a2 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Tue, 22 Jun 2021 15:07:48 -0700 Subject: [PATCH 03/16] skip-nodes as default --- changegen/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changegen/generator.py b/changegen/generator.py index d730f09..d5becf7 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -811,7 +811,7 @@ def generate_deletions( osmsrc, outfile, compress=True, - skip_nodes=False, + skip_nodes=True, ): """ Produce a changefile with nodes for all IDs in table. From a791ef307ecd8f565f523585dd08bc02312bbb0c Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 25 Jun 2021 16:51:51 -0700 Subject: [PATCH 04/16] first pass at relation handling module --- changegen/relations.py | 214 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 changegen/relations.py diff --git a/changegen/relations.py b/changegen/relations.py new file mode 100644 index 0000000..6138720 --- /dev/null +++ b/changegen/relations.py @@ -0,0 +1,214 @@ +import logging +from typing import Dict +from typing import List +from typing import Set +from typing import Union + +import osmium + +from .changewriter import Node +from .changewriter import Relation +from .changewriter import RelationMember +from .changewriter import Tag +from .changewriter import Way + + +""" + +Relation Management +==================== + +This module provides support for modifying Relations. It supports a few use-cases. + +1. A tag will be created, and some objects within it need to be + added to relations that already exist. In this case, tags are + created that modify the Relations in question. +2. more later probably + +It is important to note that this module is STATEFUL. It should probably be a class. +If you need to clear the relations DB and modified relations list, you can use _reset(). + +Insertion in to Existing Relations +----------------------------------- + +In order to insert an object into an existing relation a particular schema +of the input data is required. In particular, any object that is to be inserted +into a Relation must contain a Tag with a Key that begins with a user-specifiable +prefix and a Value that represents the ID of the Relation that the object should be +inserted into. + +The default Tag Key that is used is `_member_of`. (To use another, pass it as +the `relation_tag_prefix` argument to `get_modified_relations_for_object`). + +`modify_relations_with_object` is responsible for modifying a local +database of Relations with by objects to them, as specified in the object itself. + +Before running `modify_relations_with_object`, a local database of +Relations must be generated. This is done by providing a list of +Relation IDs to `get_relations`. This is an expensive operation, +so should be done as infrequently as possible (likely just once). + +The list of IDs provided to `get_relations` should represent all relations +that need to be inserted into. + +After processing objects is complete, call get_modified_relations +to obtain a list of modified relations that can then be used to create + tags in a changefile. + +""" + +RELATIONS_DB: Dict[str, Relation] = {} +MODIFIED_RELATIONS: Set[str] = set() + + +def _reset(): + # Used for tests currently. + global RELATIONS_DB + global MODIFIED_RELATIONS + + RELATIONS_DB = {} + MODIFIED_RELATIONS = set() + + +def get_modified_relations(): + return [RELATIONS_DB[_r] for _r in MODIFIED_RELATIONS] + + +def modify_relations_with_object( + osm_object: Union[Relation, Node, Way], relation_tag_prefix: str = "_member_of_" +) -> List[Relation]: + """ + + This function interrogates `osm_object` for Tags whose + keys begin with `relation_tag_prefix`. For all keys that begin with + that prefix, the function searches a database of relations using the values + of those tags as the relation ID. For all matching relations + we add a RelationMember to the relation representing `osm_object` + and update the database. + + Relations that are not found in RELATIONS_DB are skipped. + + NOTE that this function does not support Roles. + + NOTE that this function requires that `get_relations` is invoked + first. We need to read data from the OSM file only once. You must ensure + that the invocation of `get_relations` obtains Relation objects + for all Relations that are referred-to by Tags in the OSM objects to be + inserted. + + NOTE that this function, in addition to returning modified Relations, + also modifies the underlying relation + + """ + + # these are global because we need to update them + # with additions to Relations. Seperate + # invocations need access to the updates, so it + # has to be global. I don't love it but + # I think that's easier than having + # clients maintain state themselves? + global RELATIONS_DB + global MODIFIED_RELATIONS + + relations_in_db = 0 + try: + relations_in_db = len(RELATIONS_DB.keys()) + except NameError: + raise Exception("No Relations Database exists. Did you run get_relations?") + + if relations_in_db == 0: + raise RuntimeWarning( + ( + "There are no relations in the relations database. " + "No objects will be added to any relations. " + ) + ) + + # search for matching tags that represent relation IDs + relation_ids = [ + tag.value for tag in osm_object.tags if tag.key.startswith(relation_tag_prefix) + ] + + # update each relation with osm_object. + for relation in relation_ids: + existing_relation = None + try: + existing_relation = RELATIONS_DB[relation] + except KeyError: + logging.debug( + f"Skipping modifying relation {relation} with object {osm_object.id} because it does not exist in database." + ) + continue + + objectMember = RelationMember( + ref=osm_object.id, + type=type(osm_object) + .__name__[0] + .lower(), # Node --> 'n', Way --> 'w', 'Relation' -> 'r', + role="", + ) + + new_relation = Relation( + id=existing_relation.id, + version=existing_relation.version, + members=existing_relation.members + [objectMember], + tags=existing_relation.tags, + ) + + # update relations DB + RELATIONS_DB[relation] = new_relation + # add modified relation to set of modified relations + MODIFIED_RELATIONS.add(relation) + + +def get_relations(ids: List[str], osm_filepath: str) -> Dict[str, Relation]: + """ + Creates a global a mapping of OSM IDs to Relation objects for each relation + in the OSM file that's specified by `osm_filepath` + that is specified in `ids`. + + This function also sets the global variable `RELATIONS_DB`. + + """ + + # we need to update the module-level + # variable here, so we ensure that we're + # using global scope for this variable. + global RELATIONS_DB + + class _RelationReader(osmium.SimpleHandler): + def __init__(self, ids): + super(_RelationReader, self).__init__() + self.ids = set(ids) + self.relations: Dict[str, Relation] = {} + + def _convert_members( + self, members: osmium.osm.RelationMemberList + ) -> List[RelationMember]: + memberList: List[RelationMember] = [] + for member in members: + memberList.append( + RelationMember(ref=member.ref, type=member.type, role=member.role) + ) + return memberList + + def _convert_tags(self, tags: osmium.osm.TagList) -> List[Tag]: + tagList: List[Tag] = [] + for tag in tags: + tagList.append(Tag(key=tag.k, value=tag.v)) + return tagList + + def relation(self, r): + if str(r.id) in self.ids: + self.relations[str(r.id)] = Relation( + id=str(r.id), + version=2, + members=self._convert_members(r.members), + tags=self._convert_tags(r.tags), + ) + + _reader = _RelationReader(ids) + _reader.apply_file(osm_filepath) + + # set the global variable + RELATIONS_DB = _reader.relations From 0d606996613e2e17bc9c92686b3a25b66d0b1e56 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 25 Jun 2021 16:56:16 -0700 Subject: [PATCH 05/16] add tests --- test/test_relations.py | 74 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 test/test_relations.py diff --git a/test/test_relations.py b/test/test_relations.py new file mode 100644 index 0000000..b9a1f22 --- /dev/null +++ b/test/test_relations.py @@ -0,0 +1,74 @@ +import unittest + +from changegen import changewriter +from changegen import relations + +test_relation_id = "4567" + +test_relation_db = { + test_relation_id: changewriter.Relation( + id=test_relation_id, + version="-1", + members=[changewriter.RelationMember("-1", type="w", role="")], + tags=changewriter.Tag(key="tagkey", value="tagvalue"), + ) +} + +test_insertion_object = changewriter.Node( + id="9999", + version="-1", + lat="0", + lon="0", + tags=[changewriter.Tag(key="_member_of_somerelation", value=test_relation_id)], +) + +test_insertion_object_missing_relation = changewriter.Node( + id="9999", + version="-1", + lat="0", + lon="0", + tags=[changewriter.Tag(key="_member_of_somerelation", value="-1")], +) + + +class TestRelations(unittest.TestCase): + def test_add_node_to_relation(self): + """Ensure that a Node gets added to a Relation properly.""" + relations._reset() + ## we need to cheat and patch RELATIONS_DB with our mock + ## because I don't want to test get_relations here. + relations.RELATIONS_DB = test_relation_db + + relations.modify_relations_with_object(test_insertion_object) + + modified_relations = relations.get_modified_relations() + + self.assertEqual(len(modified_relations), 1) + + def test_proper_relation_member_formatting(self): + """Ensure that the RelationMember that's added to the Relation is proper""" + relations._reset() + ## we need to cheat and patch RELATIONS_DB with our mock + ## because I don't want to test get_relations here. + relations.RELATIONS_DB = test_relation_db + + relations.modify_relations_with_object(test_insertion_object) + + modified_relations = relations.get_modified_relations() + + self.assertTrue(modified_relations[0].members[1].type == "n") + self.assertTrue( + modified_relations[0].members[1].ref == test_insertion_object.id + ) + self.assertTrue(modified_relations[0].members[1].role == "") + + def test_relation_missing(self): + relations._reset() + + relations.RELATIONS_DB = test_relation_db + + relations.modify_relations_with_object(test_insertion_object_missing_relation) + + modified_relations = relations.get_modified_relations() + + self.assertEqual(len(modified_relations), 0) From deb0a06c6b925146170dca231a4385a7a367c762 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 25 Jun 2021 17:01:22 -0700 Subject: [PATCH 06/16] update comments --- changegen/relations.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/changegen/relations.py b/changegen/relations.py index 6138720..7caada3 100644 --- a/changegen/relations.py +++ b/changegen/relations.py @@ -86,6 +86,9 @@ def modify_relations_with_object( we add a RelationMember to the relation representing `osm_object` and update the database. + This function is not a "pure" function -- it modifies underlying state + without returning anything. + Relations that are not found in RELATIONS_DB are skipped. NOTE that this function does not support Roles. @@ -96,8 +99,7 @@ def modify_relations_with_object( for all Relations that are referred-to by Tags in the OSM objects to be inserted. - NOTE that this function, in addition to returning modified Relations, - also modifies the underlying relation + """ @@ -140,6 +142,7 @@ def modify_relations_with_object( ) continue + # create a new RelationMember containing the new object objectMember = RelationMember( ref=osm_object.id, type=type(osm_object) @@ -148,6 +151,7 @@ def modify_relations_with_object( role="", ) + # create a new relation containing the new member. new_relation = Relation( id=existing_relation.id, version=existing_relation.version, @@ -163,12 +167,10 @@ def modify_relations_with_object( def get_relations(ids: List[str], osm_filepath: str) -> Dict[str, Relation]: """ - Creates a global a mapping of OSM IDs to Relation objects for each relation + Creates an internal mapping of OSM IDs to Relation objects for each relation in the OSM file that's specified by `osm_filepath` that is specified in `ids`. - This function also sets the global variable `RELATIONS_DB`. - """ # we need to update the module-level From 86143a36869aebf8c0f8c1980f5ed17624ed9025 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 25 Jun 2021 17:46:20 -0700 Subject: [PATCH 07/16] update relations: modify db with objects, add modify nodes, add command line support --- changegen/__main__.py | 23 +++++++++++++++++++++++ changegen/generator.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/changegen/__main__.py b/changegen/__main__.py index dfaf71a..eb4ded6 100644 --- a/changegen/__main__.py +++ b/changegen/__main__.py @@ -147,6 +147,27 @@ def _get_db_tables(suffix, dbname, dbport, dbuser, dbpass, dbhost): ), default="2000", ) +@click.option( + "--modify_relations", + is_flag=True, + help=( + "Add new objects to parent relations specified by a special tag. " + 'The default tag prefix for tags containing Relation IDs is "_member_of_". ' + "Pass the --relation_member_prefix flag to change " + "this prefix, e.g. --relation_member_prefix __a_different_prefix_. " + "See changegen.relations.py for more information. " + ), +) +@click.option( + "--relation_member_prefix", + is_flag=False, + help=( + "Only used with --modify_relations. Specify the tag prefix " + "used to search for IDs to to add new OSM objects to." + ), + default="_member_of_", + show_default=True, +) @click.option("--osmsrc", help="Source OSM PBF File path", required=True) @click.argument("dbname", default=os.environ.get("PGDATABASE", "conflate")) @click.argument("dbport", default=os.environ.get("PGPORT", "15432")) @@ -233,6 +254,8 @@ def main(*args: tuple, **kwargs: dict): self_intersections=kwargs["self"], max_nodes_per_way=int(max_nodes_per_way), modify_only=kwargs["modify_meta"], + modify_relations=kwargs["modify_relations"], + relation_member_prefix=kwargs["relation_member_prefix"], ) for table in kwargs["deletions"]: diff --git a/changegen/generator.py b/changegen/generator.py index b6b915e..fbc22d5 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -24,6 +24,9 @@ from .changewriter import Tag from .changewriter import Way from .db import OGRDBReader +from .relations import get_modified_relations +from .relations import get_relations +from .relations import modify_relations_with_object WGS84 = pyproj.CRS("EPSG:4326") WEBMERC = pyproj.CRS("EPSG:3857") @@ -483,6 +486,8 @@ def generate_changes( self_intersections=False, max_nodes_per_way=2000, modify_only=False, + modify_relations=False, + relation_member_prefix="_member_of", ): """ Generate an osm changefile (outfile) based on features in @@ -676,6 +681,32 @@ def generate_changes( else: raise RuntimeError(f"{type(wgs84_geom)} is not LineString or Polygon") + ## Relation Updates: If modify_relations is true, + ## we'll search through all newly-added objects + ## for Tags with the "relation_member" prefix + ## and add them to the relations specified by the + ## values of those tags. + modified_relations = [] + if modify_relations: + relations_mentioned = set() + for obj in new_ways + new_nodes + new_relations: + relations_mentioned.update( + [ + _t.value + for _t in obj.tags + if _t.key.startswith(relation_member_prefix) + ] + ) + # create relations DB + get_relations(relations_mentioned, osmsrc) + # update db for each new object + for obj in tqdm( + new_ways + new_nodes + new_relations, desc="Updating relations..." + ): + modify_relations_with_object(obj, relation_member_prefix) + # get modified relations + modified_relations = get_modified_relations() + ## Write new ways and nodes to file if len(new_ways) > 0 or len(new_nodes) > 0: if modify_only: @@ -684,6 +715,9 @@ def generate_changes( change_writer.add_create(new_nodes + new_ways) if len(new_relations) > 0: change_writer.add_create(new_relations) + ## Write modified relations too. + if modify_relations and len(modified_relations) > 0: + change_writer.add_modify(modified_relations) # Write all modified ways with intersections # Because we have to re-generate nodes for all points From a9da4414083988ad526ec83d2cfcc78ae55b42af Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 25 Jun 2021 17:47:50 -0700 Subject: [PATCH 08/16] clarify comment --- changegen/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changegen/generator.py b/changegen/generator.py index fbc22d5..ce79e49 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -683,7 +683,7 @@ def generate_changes( ## Relation Updates: If modify_relations is true, ## we'll search through all newly-added objects - ## for Tags with the "relation_member" prefix + ## for Tags with prefix specified by `relation_member_prefix` ## and add them to the relations specified by the ## values of those tags. modified_relations = [] From 32907b6b96a0e1620fda26e6070d0fb16f4ac2e7 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Mon, 28 Jun 2021 15:51:51 -0700 Subject: [PATCH 09/16] refactor relations control flow out of individual object process --- changegen/generator.py | 55 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/changegen/generator.py b/changegen/generator.py index b80d327..8b25048 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -696,32 +696,6 @@ def generate_changes( else: raise RuntimeError(f"{type(wgs84_geom)} is not LineString or Polygon") - ## Relation Updates: If modify_relations is true, - ## we'll search through all newly-added objects - ## for Tags with prefix specified by `relation_member_prefix` - ## and add them to the relations specified by the - ## values of those tags. - modified_relations = [] - if modify_relations: - relations_mentioned = set() - for obj in new_ways + new_nodes + new_relations: - relations_mentioned.update( - [ - _t.value - for _t in obj.tags - if _t.key.startswith(relation_member_prefix) - ] - ) - # create relations DB - get_relations(relations_mentioned, osmsrc) - # update db for each new object - for obj in tqdm( - new_ways + new_nodes + new_relations, desc="Updating relations..." - ): - modify_relations_with_object(obj, relation_member_prefix) - # get modified relations - modified_relations = get_modified_relations() - ## Write new ways and nodes to file if len(new_ways) > 0 or len(new_nodes) > 0: if modify_only: @@ -730,8 +704,35 @@ def generate_changes( change_writer.add_create(new_nodes + new_ways) if len(new_relations) > 0: change_writer.add_create(new_relations) + + ## Relation Updates: If modify_relations is true, + ## we'll search through all newly-added objects + ## for Tags with prefix specified by `relation_member_prefix` + ## and add them to the relations specified by the + ## values of those tags. + modified_relations = [] + if modify_relations: + relations_mentioned = set() + for obj in chain(new_ways, new_nodes, new_relations): + relations_mentioned.update( + [ + _t.value + for _t in obj.tags + if _t.key.startswith(relation_member_prefix) + ] + ) + # create relations DB + get_relations(relations_mentioned, osmsrc) + # update db for each new object + for obj in tqdm( + new_ways + new_nodes + new_relations, + desc="Checking objects for relations...", + ): + modify_relations_with_object(obj, relation_member_prefix) + # get modified relations + modified_relations = get_modified_relations() ## Write modified relations too. - if modify_relations and len(modified_relations) > 0: + if len(modified_relations) > 0: change_writer.add_modify(modified_relations) # Write all modified ways with intersections From 0cfa8ef65c9475abb67b645f3efc83740e91d446 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Wed, 30 Jun 2021 08:35:00 -0600 Subject: [PATCH 10/16] need full relation member type --- changegen/relations.py | 6 +++--- test/test_relations.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/changegen/relations.py b/changegen/relations.py index 7caada3..77fd8a0 100644 --- a/changegen/relations.py +++ b/changegen/relations.py @@ -145,9 +145,9 @@ def modify_relations_with_object( # create a new RelationMember containing the new object objectMember = RelationMember( ref=osm_object.id, - type=type(osm_object) - .__name__[0] - .lower(), # Node --> 'n', Way --> 'w', 'Relation' -> 'r', + type=type( + osm_object + ).__name__.lower(), # Node --> 'node', Way --> 'way', 'Relation' -> 'relation', role="", ) diff --git a/test/test_relations.py b/test/test_relations.py index b9a1f22..8cc9a2a 100644 --- a/test/test_relations.py +++ b/test/test_relations.py @@ -9,7 +9,7 @@ test_relation_id: changewriter.Relation( id=test_relation_id, version="-1", - members=[changewriter.RelationMember("-1", type="w", role="")], + members=[changewriter.RelationMember("-1", type="way", role="")], tags=changewriter.Tag(key="tagkey", value="tagvalue"), ) } @@ -56,7 +56,7 @@ def test_proper_relation_member_formatting(self): modified_relations = relations.get_modified_relations() - self.assertTrue(modified_relations[0].members[1].type == "n") + self.assertTrue(modified_relations[0].members[1].type == "node") self.assertTrue( modified_relations[0].members[1].ref == test_insertion_object.id ) From c8a731a139e985fa2fe4b596b13f095cf48ec848 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Wed, 30 Jun 2021 11:17:31 -0600 Subject: [PATCH 11/16] ensure releations from file use osmium-compatible types --- changegen/relations.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/changegen/relations.py b/changegen/relations.py index 77fd8a0..77aded1 100644 --- a/changegen/relations.py +++ b/changegen/relations.py @@ -12,6 +12,9 @@ from .changewriter import Tag from .changewriter import Way +# whether to use "way" or "w" (etc) for +# the "type" field of RelationMembers. +LONG_RELATION_MEMBER_TYPE = True """ @@ -143,11 +146,12 @@ def modify_relations_with_object( continue # create a new RelationMember containing the new object + rm_type = type(osm_object).__name__.lower() + if not LONG_RELATION_MEMBER_TYPE: + rm_type = rm_type[0] objectMember = RelationMember( ref=osm_object.id, - type=type( - osm_object - ).__name__.lower(), # Node --> 'node', Way --> 'way', 'Relation' -> 'relation', + type=rm_type, role="", ) @@ -189,8 +193,15 @@ def _convert_members( ) -> List[RelationMember]: memberList: List[RelationMember] = [] for member in members: + _type = member.type + if LONG_RELATION_MEMBER_TYPE: + _type = { + "w": "way", + "n": "node", + "r": "relation", + }[member.type] memberList.append( - RelationMember(ref=member.ref, type=member.type, role=member.role) + RelationMember(ref=member.ref, type=_type, role=member.role) ) return memberList From 1904e61ac84aada04e7ea84a202b3c441f6963e3 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 2 Jul 2021 08:27:04 -0700 Subject: [PATCH 12/16] make relation updater a class --- changegen/generator.py | 11 +- changegen/relations.py | 278 +++++++++++++++++++---------------------- test/test_relations.py | 44 +++++-- 3 files changed, 169 insertions(+), 164 deletions(-) diff --git a/changegen/generator.py b/changegen/generator.py index 8b25048..e727b84 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -24,9 +24,7 @@ from .changewriter import Tag from .changewriter import Way from .db import OGRDBReader -from .relations import get_modified_relations -from .relations import get_relations -from .relations import modify_relations_with_object +from .relations import RelationUpdater WGS84 = pyproj.CRS("EPSG:4326") WEBMERC = pyproj.CRS("EPSG:3857") @@ -712,6 +710,7 @@ def generate_changes( ## values of those tags. modified_relations = [] if modify_relations: + updater = RelationUpdater() relations_mentioned = set() for obj in chain(new_ways, new_nodes, new_relations): relations_mentioned.update( @@ -722,15 +721,15 @@ def generate_changes( ] ) # create relations DB - get_relations(relations_mentioned, osmsrc) + updater.get_relations(relations_mentioned, osmsrc) # update db for each new object for obj in tqdm( new_ways + new_nodes + new_relations, desc="Checking objects for relations...", ): - modify_relations_with_object(obj, relation_member_prefix) + updater.modify_relations_with_object(obj, relation_member_prefix) # get modified relations - modified_relations = get_modified_relations() + modified_relations = updater.get_modified_relations() ## Write modified relations too. if len(modified_relations) > 0: change_writer.add_modify(modified_relations) diff --git a/changegen/relations.py b/changegen/relations.py index 77aded1..533532f 100644 --- a/changegen/relations.py +++ b/changegen/relations.py @@ -60,168 +60,154 @@ """ -RELATIONS_DB: Dict[str, Relation] = {} -MODIFIED_RELATIONS: Set[str] = set() +class RelationUpdater(object): + def __init__(self): + self.RELATIONS_DB: Dict[str, Relation] = {} + self.MODIFIED_RELATIONS: Set[str] = set() -def _reset(): - # Used for tests currently. - global RELATIONS_DB - global MODIFIED_RELATIONS + def _reset(self): + # Used for tests currently. - RELATIONS_DB = {} - MODIFIED_RELATIONS = set() + self.RELATIONS_DB = {} + self.MODIFIED_RELATIONS = set() + def get_modified_relations(self): + return [self.RELATIONS_DB[_r] for _r in self.MODIFIED_RELATIONS] -def get_modified_relations(): - return [RELATIONS_DB[_r] for _r in MODIFIED_RELATIONS] + def modify_relations_with_object( + self, + osm_object: Union[Relation, Node, Way], + relation_tag_prefix: str = "_member_of_", + ) -> List[Relation]: + """ + This function interrogates `osm_object` for Tags whose + keys begin with `relation_tag_prefix`. For all keys that begin with + that prefix, the function searches a database of relations using the values + of those tags as the relation ID. For all matching relations + we add a RelationMember to the relation representing `osm_object` + and update the database. -def modify_relations_with_object( - osm_object: Union[Relation, Node, Way], relation_tag_prefix: str = "_member_of_" -) -> List[Relation]: - """ + This function is not a "pure" function -- it modifies underlying state + without returning anything. - This function interrogates `osm_object` for Tags whose - keys begin with `relation_tag_prefix`. For all keys that begin with - that prefix, the function searches a database of relations using the values - of those tags as the relation ID. For all matching relations - we add a RelationMember to the relation representing `osm_object` - and update the database. + Relations that are not found in RELATIONS_DB are skipped. - This function is not a "pure" function -- it modifies underlying state - without returning anything. + NOTE that this function does not support Roles. - Relations that are not found in RELATIONS_DB are skipped. + NOTE that this function requires that `get_relations` is invoked + first. We need to read data from the OSM file only once. You must ensure + that the invocation of `get_relations` obtains Relation objects + for all Relations that are referred-to by Tags in the OSM objects to be + inserted. - NOTE that this function does not support Roles. - NOTE that this function requires that `get_relations` is invoked - first. We need to read data from the OSM file only once. You must ensure - that the invocation of `get_relations` obtains Relation objects - for all Relations that are referred-to by Tags in the OSM objects to be - inserted. + """ - - """ - - # these are global because we need to update them - # with additions to Relations. Seperate - # invocations need access to the updates, so it - # has to be global. I don't love it but - # I think that's easier than having - # clients maintain state themselves? - global RELATIONS_DB - global MODIFIED_RELATIONS - - relations_in_db = 0 - try: - relations_in_db = len(RELATIONS_DB.keys()) - except NameError: - raise Exception("No Relations Database exists. Did you run get_relations?") - - if relations_in_db == 0: - raise RuntimeWarning( - ( - "There are no relations in the relations database. " - "No objects will be added to any relations. " - ) - ) - - # search for matching tags that represent relation IDs - relation_ids = [ - tag.value for tag in osm_object.tags if tag.key.startswith(relation_tag_prefix) - ] - - # update each relation with osm_object. - for relation in relation_ids: - existing_relation = None + relations_in_db = 0 try: - existing_relation = RELATIONS_DB[relation] - except KeyError: - logging.debug( - f"Skipping modifying relation {relation} with object {osm_object.id} because it does not exist in database." - ) - continue - - # create a new RelationMember containing the new object - rm_type = type(osm_object).__name__.lower() - if not LONG_RELATION_MEMBER_TYPE: - rm_type = rm_type[0] - objectMember = RelationMember( - ref=osm_object.id, - type=rm_type, - role="", - ) - - # create a new relation containing the new member. - new_relation = Relation( - id=existing_relation.id, - version=existing_relation.version, - members=existing_relation.members + [objectMember], - tags=existing_relation.tags, - ) - - # update relations DB - RELATIONS_DB[relation] = new_relation - # add modified relation to set of modified relations - MODIFIED_RELATIONS.add(relation) - - -def get_relations(ids: List[str], osm_filepath: str) -> Dict[str, Relation]: - """ - Creates an internal mapping of OSM IDs to Relation objects for each relation - in the OSM file that's specified by `osm_filepath` - that is specified in `ids`. - - """ - - # we need to update the module-level - # variable here, so we ensure that we're - # using global scope for this variable. - global RELATIONS_DB - - class _RelationReader(osmium.SimpleHandler): - def __init__(self, ids): - super(_RelationReader, self).__init__() - self.ids = set(ids) - self.relations: Dict[str, Relation] = {} - - def _convert_members( - self, members: osmium.osm.RelationMemberList - ) -> List[RelationMember]: - memberList: List[RelationMember] = [] - for member in members: - _type = member.type - if LONG_RELATION_MEMBER_TYPE: - _type = { - "w": "way", - "n": "node", - "r": "relation", - }[member.type] - memberList.append( - RelationMember(ref=member.ref, type=_type, role=member.role) + relations_in_db = len(self.RELATIONS_DB.keys()) + except NameError: + raise Exception("No Relations Database exists. Did you run get_relations?") + + if relations_in_db == 0: + raise RuntimeWarning( + ( + "There are no relations in the relations database. " + "No objects will be added to any relations. " ) - return memberList - - def _convert_tags(self, tags: osmium.osm.TagList) -> List[Tag]: - tagList: List[Tag] = [] - for tag in tags: - tagList.append(Tag(key=tag.k, value=tag.v)) - return tagList - - def relation(self, r): - if str(r.id) in self.ids: - self.relations[str(r.id)] = Relation( - id=str(r.id), - version=2, - members=self._convert_members(r.members), - tags=self._convert_tags(r.tags), + ) + + # search for matching tags that represent relation IDs + relation_ids = [ + tag.value + for tag in osm_object.tags + if tag.key.startswith(relation_tag_prefix) + ] + + # update each relation with osm_object. + for relation in relation_ids: + existing_relation = None + try: + existing_relation = self.RELATIONS_DB[relation] + except KeyError: + logging.debug( + f"Skipping modifying relation {relation} with object {osm_object.id} because it does not exist in database." ) + continue + + # create a new RelationMember containing the new object + rm_type = type(osm_object).__name__.lower() + if not LONG_RELATION_MEMBER_TYPE: + rm_type = rm_type[0] + objectMember = RelationMember( + ref=osm_object.id, + type=rm_type, + role="", + ) - _reader = _RelationReader(ids) - _reader.apply_file(osm_filepath) + # create a new relation containing the new member. + new_relation = Relation( + id=existing_relation.id, + version=existing_relation.version, + members=existing_relation.members + [objectMember], + tags=existing_relation.tags, + ) - # set the global variable - RELATIONS_DB = _reader.relations + # update relations DB + self.RELATIONS_DB.update({relation: new_relation}) + # add modified relation to set of modified relations + self.MODIFIED_RELATIONS.add(relation) + + def get_relations(self, ids: List[str], osm_filepath: str) -> Dict[str, Relation]: + """ + Creates an internal mapping of OSM IDs to Relation objects for each relation + in the OSM file that's specified by `osm_filepath` + that is specified in `ids`. + + """ + + class _RelationReader(osmium.SimpleHandler): + def __init__(__self, ids): + super(_RelationReader, __self).__init__() + __self.ids = set(ids) + __self.relations: Dict[str, Relation] = {} + + def _convert_members( + __self, members: osmium.osm.RelationMemberList + ) -> List[RelationMember]: + memberList: List[RelationMember] = [] + for member in members: + _type = member.type + if LONG_RELATION_MEMBER_TYPE: + _type = { + "w": "way", + "n": "node", + "r": "relation", + }[member.type] + memberList.append( + RelationMember(ref=member.ref, type=_type, role=member.role) + ) + return memberList + + def _convert_tags(__self, tags: osmium.osm.TagList) -> List[Tag]: + tagList: List[Tag] = [] + for tag in tags: + tagList.append(Tag(key=tag.k, value=tag.v)) + return tagList + + def relation(__self, r): + if str(r.id) in self.ids: + __self.relations[str(r.id)] = Relation( + id=str(r.id), + version=2, + members=__self._convert_members(r.members), + tags=__self._convert_tags(r.tags), + ) + + _reader = _RelationReader(ids) + _reader.apply_file(osm_filepath) + + self.RELATIONS_DB = _reader.relations diff --git a/test/test_relations.py b/test/test_relations.py index 8cc9a2a..6d9545d 100644 --- a/test/test_relations.py +++ b/test/test_relations.py @@ -22,6 +22,14 @@ tags=[changewriter.Tag(key="_member_of_somerelation", value=test_relation_id)], ) +another_test_insertion_object = changewriter.Node( + id="9998", + version="-1", + lat="0", + lon="0", + tags=[changewriter.Tag(key="_member_of_somerelation", value=test_relation_id)], +) + test_insertion_object_missing_relation = changewriter.Node( id="9999", version="-1", @@ -34,27 +42,39 @@ class TestRelations(unittest.TestCase): def test_add_node_to_relation(self): """Ensure that a Node gets added to a Relation properly.""" - relations._reset() + ru = relations.RelationUpdater() ## we need to cheat and patch RELATIONS_DB with our mock ## because I don't want to test get_relations here. - relations.RELATIONS_DB = test_relation_db + ru.RELATIONS_DB = test_relation_db - relations.modify_relations_with_object(test_insertion_object) + ru.modify_relations_with_object(test_insertion_object) - modified_relations = relations.get_modified_relations() + modified_relations = ru.get_modified_relations() self.assertEqual(len(modified_relations), 1) + def test_add_multiple(self): + """Ensure that adding multiple ways to relation works.""" + ru = relations.RelationUpdater() + ru.RELATIONS_DB = test_relation_db + + ru.modify_relations_with_object(test_insertion_object) + ru.modify_relations_with_object(another_test_insertion_object) + + modified_relations = ru.get_modified_relations() + + self.assertEqual(len(modified_relations[0].members), 3) + def test_proper_relation_member_formatting(self): """Ensure that the RelationMember that's added to the Relation is proper""" - relations._reset() + ru = relations.RelationUpdater() ## we need to cheat and patch RELATIONS_DB with our mock ## because I don't want to test get_relations here. - relations.RELATIONS_DB = test_relation_db + ru.RELATIONS_DB = test_relation_db - relations.modify_relations_with_object(test_insertion_object) + ru.modify_relations_with_object(test_insertion_object) - modified_relations = relations.get_modified_relations() + modified_relations = ru.get_modified_relations() self.assertTrue(modified_relations[0].members[1].type == "node") self.assertTrue( @@ -63,12 +83,12 @@ def test_proper_relation_member_formatting(self): self.assertTrue(modified_relations[0].members[1].role == "") def test_relation_missing(self): - relations._reset() + ru = relations.RelationUpdater() - relations.RELATIONS_DB = test_relation_db + ru.RELATIONS_DB = test_relation_db - relations.modify_relations_with_object(test_insertion_object_missing_relation) + ru.modify_relations_with_object(test_insertion_object_missing_relation) - modified_relations = relations.get_modified_relations() + modified_relations = ru.get_modified_relations() self.assertEqual(len(modified_relations), 0) From be786624ee573fe0852282dd125b8deb89d14e6d Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 2 Jul 2021 08:34:08 -0700 Subject: [PATCH 13/16] typo --- changegen/relations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changegen/relations.py b/changegen/relations.py index 533532f..3c4cb96 100644 --- a/changegen/relations.py +++ b/changegen/relations.py @@ -199,7 +199,7 @@ def _convert_tags(__self, tags: osmium.osm.TagList) -> List[Tag]: return tagList def relation(__self, r): - if str(r.id) in self.ids: + if str(r.id) in __self.ids: __self.relations[str(r.id)] = Relation( id=str(r.id), version=2, From 61026970fb7228d305c72dbc510045aea52e6a6b Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 2 Jul 2021 09:02:47 -0700 Subject: [PATCH 14/16] fix issue with only 1 relation updating --- changegen/changewriter.py | 12 +++++++++++- changegen/generator.py | 10 +++++++--- changegen/relations.py | 1 - 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/changegen/changewriter.py b/changegen/changewriter.py index 672244f..828ca28 100644 --- a/changegen/changewriter.py +++ b/changegen/changewriter.py @@ -107,7 +107,7 @@ class OSMChangeWriter(object): ) _root_element_close = "" - def __init__(self, filename=None, compress=False): + def __init__(self, filename=None, compress=False, keepcopy=False): super(OSMChangeWriter, self).__init__() self.compress = compress @@ -115,6 +115,10 @@ def __init__(self, filename=None, compress=False): self.fileobj = None self.closed = False self._data_written = False + self.keepcopy = keepcopy + self.created = [] + self.modified = [] + self.deleted = [] # set fileobj based on compression if self.filename and self.compress: @@ -163,6 +167,8 @@ def add_modify(self, elementlist): write_osm_object(e, writer) writer.flush() self._data_written = True + if self.keepcopy: + self.modified.extend(elementlist) def add_create(self, elementlist): """Creates element containing @@ -178,6 +184,8 @@ def add_create(self, elementlist): write_osm_object(e, writer) writer.flush() self._data_written = True + if self.keepcopy: + self.created.extend(elementlist) def add_delete(self, elementlist): """Creates a element containing @@ -189,3 +197,5 @@ def add_delete(self, elementlist): write_osm_object(e, writer) writer.flush() self._data_written = True + if self.keepcopy: + self.deleted.extend(elementlist) diff --git a/changegen/generator.py b/changegen/generator.py index e727b84..30adc79 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -517,7 +517,11 @@ def generate_changes( others = [others] if isinstance(others, str) else others db_reader = OGRDBReader(dbname, dbport, dbuser, dbpass, dbhost) - change_writer = OSMChangeWriter(outfile, compress=compress) + # keep a copy of elements if we modify_relations + # becasue we need them later + change_writer = OSMChangeWriter( + outfile, compress=compress, keepcopy=modify_relations + ) new_feature_iter = db_reader.get_layer_iter(table) layer_fields = db_reader.get_layer_fields(table) @@ -712,7 +716,7 @@ def generate_changes( if modify_relations: updater = RelationUpdater() relations_mentioned = set() - for obj in chain(new_ways, new_nodes, new_relations): + for obj in change_writer.created: relations_mentioned.update( [ _t.value @@ -724,7 +728,7 @@ def generate_changes( updater.get_relations(relations_mentioned, osmsrc) # update db for each new object for obj in tqdm( - new_ways + new_nodes + new_relations, + change_writer.created, desc="Checking objects for relations...", ): updater.modify_relations_with_object(obj, relation_member_prefix) diff --git a/changegen/relations.py b/changegen/relations.py index 3c4cb96..91f18b0 100644 --- a/changegen/relations.py +++ b/changegen/relations.py @@ -126,7 +126,6 @@ def modify_relations_with_object( for tag in osm_object.tags if tag.key.startswith(relation_tag_prefix) ] - # update each relation with osm_object. for relation in relation_ids: existing_relation = None From e3f75dd8cdc9eb1aa0c627052a4ece1cae30a52d Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Fri, 2 Jul 2021 12:46:23 -0700 Subject: [PATCH 15/16] use comma-separated string instead of multiple tags --- changegen/__main__.py | 12 ++++++------ changegen/generator.py | 18 ++++++++++-------- changegen/relations.py | 31 +++++++++++++++++-------------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/changegen/__main__.py b/changegen/__main__.py index eb4ded6..349f4bb 100644 --- a/changegen/__main__.py +++ b/changegen/__main__.py @@ -152,20 +152,20 @@ def _get_db_tables(suffix, dbname, dbport, dbuser, dbpass, dbhost): is_flag=True, help=( "Add new objects to parent relations specified by a special tag. " - 'The default tag prefix for tags containing Relation IDs is "_member_of_". ' - "Pass the --relation_member_prefix flag to change " - "this prefix, e.g. --relation_member_prefix __a_different_prefix_. " + 'The default tag prefix for tags containing Relation IDs is "_member_of". ' + "Pass the --relation_tag flag to change " + "this prefix, e.g. --relation_tag __a_different_prefix_. " "See changegen.relations.py for more information. " ), ) @click.option( - "--relation_member_prefix", + "--relation_tag", is_flag=False, help=( "Only used with --modify_relations. Specify the tag prefix " "used to search for IDs to to add new OSM objects to." ), - default="_member_of_", + default="_member_of", show_default=True, ) @click.option("--osmsrc", help="Source OSM PBF File path", required=True) @@ -255,7 +255,7 @@ def main(*args: tuple, **kwargs: dict): max_nodes_per_way=int(max_nodes_per_way), modify_only=kwargs["modify_meta"], modify_relations=kwargs["modify_relations"], - relation_member_prefix=kwargs["relation_member_prefix"], + relation_tag=kwargs["relation_tag"], ) for table in kwargs["deletions"]: diff --git a/changegen/generator.py b/changegen/generator.py index 30adc79..98f8a74 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -485,7 +485,7 @@ def generate_changes( max_nodes_per_way=2000, modify_only=False, modify_relations=False, - relation_member_prefix="_member_of", + relation_tag="_member_of", ): """ Generate an osm changefile (outfile) based on features in
@@ -709,7 +709,7 @@ def generate_changes( ## Relation Updates: If modify_relations is true, ## we'll search through all newly-added objects - ## for Tags with prefix specified by `relation_member_prefix` + ## for Tags with prefix specified by `relation_tag` ## and add them to the relations specified by the ## values of those tags. modified_relations = [] @@ -718,11 +718,13 @@ def generate_changes( relations_mentioned = set() for obj in change_writer.created: relations_mentioned.update( - [ - _t.value - for _t in obj.tags - if _t.key.startswith(relation_member_prefix) - ] + chain.from_iterable( + [ + _t.value.split(",") + for _t in obj.tags + if _t.key.startswith(relation_tag) + ] + ) ) # create relations DB updater.get_relations(relations_mentioned, osmsrc) @@ -731,7 +733,7 @@ def generate_changes( change_writer.created, desc="Checking objects for relations...", ): - updater.modify_relations_with_object(obj, relation_member_prefix) + updater.modify_relations_with_object(obj, relation_tag) # get modified relations modified_relations = updater.get_modified_relations() ## Write modified relations too. diff --git a/changegen/relations.py b/changegen/relations.py index 91f18b0..b2bab77 100644 --- a/changegen/relations.py +++ b/changegen/relations.py @@ -1,4 +1,5 @@ import logging +from itertools import chain from typing import Dict from typing import List from typing import Set @@ -36,12 +37,12 @@ In order to insert an object into an existing relation a particular schema of the input data is required. In particular, any object that is to be inserted -into a Relation must contain a Tag with a Key that begins with a user-specifiable -prefix and a Value that represents the ID of the Relation that the object should be -inserted into. +into a Relation must contain a Tag with a user-specifiable Key +and a Value that represents a comma-separated list of the IDs of +Relations that the object should be inserted into. The default Tag Key that is used is `_member_of`. (To use another, pass it as -the `relation_tag_prefix` argument to `get_modified_relations_for_object`). +the `relation_tag` argument to `get_modified_relations_for_object`). `modify_relations_with_object` is responsible for modifying a local database of Relations with by objects to them, as specified in the object itself. @@ -78,14 +79,14 @@ def get_modified_relations(self): def modify_relations_with_object( self, osm_object: Union[Relation, Node, Way], - relation_tag_prefix: str = "_member_of_", + relation_tag: str = "_member_of", ) -> List[Relation]: """ - This function interrogates `osm_object` for Tags whose - keys begin with `relation_tag_prefix`. For all keys that begin with - that prefix, the function searches a database of relations using the values - of those tags as the relation ID. For all matching relations + This function interrogates `osm_object` for a Tag whose + key begins with `relation_tag`. If a matching key is found that begins with + that prefix, the function searches a database of relations using the comma-separated + values in the Tag as the relation IDs. For all matching relations we add a RelationMember to the relation representing `osm_object` and update the database. @@ -121,11 +122,13 @@ def modify_relations_with_object( ) # search for matching tags that represent relation IDs - relation_ids = [ - tag.value - for tag in osm_object.tags - if tag.key.startswith(relation_tag_prefix) - ] + relation_ids = chain.from_iterable( + [ + tag.value.split(",") + for tag in osm_object.tags + if tag.key.startswith(relation_tag) + ] + ) # update each relation with osm_object. for relation in relation_ids: existing_relation = None From 73830db7e2426eee3e4c4e061a5d11f79dd39e17 Mon Sep 17 00:00:00 2001 From: Tony Cannistra Date: Mon, 5 Jul 2021 18:52:16 -0700 Subject: [PATCH 16/16] add support for mid-insertion of relation members --- changegen/generator.py | 17 ++++++++++++++++- changegen/relations.py | 39 ++++++++++++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/changegen/generator.py b/changegen/generator.py index 98f8a74..12b5a1b 100644 --- a/changegen/generator.py +++ b/changegen/generator.py @@ -486,6 +486,7 @@ def generate_changes( modify_only=False, modify_relations=False, relation_tag="_member_of", + relation_insertion_tag="parent_osm_id", ): """ Generate an osm changefile (outfile) based on features in
@@ -733,7 +734,21 @@ def generate_changes( change_writer.created, desc="Checking objects for relations...", ): - updater.modify_relations_with_object(obj, relation_tag) + # check to see if a tag that matches `relation_insertion_tag` + # is present, and if so, provide the value as` + # at_id to insert the new object at the location of that + # ID in the relation. + at_id = None + try: + at_id = [ + t.value + for t in obj.tags + if t.key.startswith(relation_insertion_tag) + ][0] + except IndexError: + # did not find insertion tag. at_id is none. + at_id = None + updater.modify_relations_with_object(obj, relation_tag, at_id) # get modified relations modified_relations = updater.get_modified_relations() ## Write modified relations too. diff --git a/changegen/relations.py b/changegen/relations.py index b2bab77..5ba2ac3 100644 --- a/changegen/relations.py +++ b/changegen/relations.py @@ -80,15 +80,22 @@ def modify_relations_with_object( self, osm_object: Union[Relation, Node, Way], relation_tag: str = "_member_of", + at_id: str = None, ) -> List[Relation]: """ This function interrogates `osm_object` for a Tag whose key begins with `relation_tag`. If a matching key is found that begins with - that prefix, the function searches a database of relations using the comma-separated - values in the Tag as the relation IDs. For all matching relations - we add a RelationMember to the relation representing `osm_object` - and update the database. + that prefix, the function: + * searches a database of relations using the comma-separated + values in the Tag as the relation IDs. + * For all matching relations we add a RelationMember to the relation representing \ + `osm_object` and update the database. + * if ``at_id`` is ``None``, we add the ``RelationMember`` to the end of the relation. + * if ``at_id`` has a value that matches an OSM ID that currently exists \ + in the relation, we add the ``RelationMember`` that represents \ + ``osm_object`` _after_ that index. + This function is not a "pure" function -- it modifies underlying state without returning anything. @@ -150,11 +157,33 @@ def modify_relations_with_object( role="", ) + # Determine where to insert the new object in the list + # of relation members. If at_id is provided, we look + # for the index of that ID in the members list and insert + # the new object at that index. If it's None, we insert it + # at the end. + relation_members = existing_relation.members + insertion_index = len(relation_members) + if at_id: + # get index of RelationMember whose id equals at_id + try: + insertion_index = next( + idx + for idx, rm in enumerate(relation_members) + if (lambda x: x.ref == at_id)(rm) + ) + except StopIteration: + logging.debug( + f"Could not find ID {at_id} in list of members of relation {existing_relation.id}." + ) + + relation_members.insert(insertion_index, objectMember) + # create a new relation containing the new member. new_relation = Relation( id=existing_relation.id, version=existing_relation.version, - members=existing_relation.members + [objectMember], + members=relation_members, tags=existing_relation.tags, )