Support PostGIS as alternative data source to Overpass

Dockerfile

@@ -4,9 +4,9 @@ COPY app requirements.txt /app
 WORKDIR /app
 RUN pip install -r requirements.txt
 
-ENV OVERPASS_URL=https://overpass.kumi.systems/api/interpreter/
+ENV BACKEND_URL=https://overpass.kumi.systems/api/interpreter/
 # Leave these variables undefined; to use sentry, provide them in a .env file with docker compose or on the command line.
 ENV SENTRY_DSN
 ENV SENTRY_TSR
 EXPOSE 8080
-CMD python main.py --overpass-url $OVERPASS_URL --sentry-dsn $SENTRY_DSN --sentry-tsr $SENTRY_TSR
+CMD python main.py --backend-url $BACKEND_URL --sentry-dsn $SENTRY_DSN --sentry-tsr $SENTRY_TSR

README.md

@@ -26,6 +26,8 @@ It serves map data by sending queries to a public or privately-hosted [Overpass]
 
 You can also run the original Soundscape server code as provided by Microsoft. Unlike Overscape, the Microsoft version involves loading and hosting of bulk OpenStreetMap data in a PostGIS database. See the [docker-compose file ](https://github.com/openscape-community/openscape/blob/master/svcs/data/docker-compose.yml) for details on spinning up the necessary services.
 
+Overscape also supports using a PostGIS server as a backend -- simply pass the argument `--backend-url postgres://user:password@host:port/db` when launching the server.
+
 ## Running tests
 ```
 pip install -r requirements_test.txt

app/main.py

@@ -7,8 +7,8 @@
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--overpass-url",
-        help="URL of Overpass API server",
+        "--backend-url",
+        help="URL of Overpass API or PostGIS server",
         default="https://overpass.kumi.systems/api/interpreter/",
         # default="http://overpass-api.de/api/interpreter",
     )
@@ -21,7 +21,7 @@
         "--cache-days",
         type=int,
         help="Number of days after which cached items should be refreshed",
-        default=7,
+        default=30,
     )
     parser.add_argument(
         "--cache-dir",
@@ -58,7 +58,7 @@
 
     logging.basicConfig(level=args.log_level)
     run_server(
-        args.overpass_url,
+        args.backend_url,
         args.user_agent,
         args.cache_dir,
         args.cache_days,

app/postgis.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+# aiopg bug workaround from https://github.com/aio-libs/aiopg/issues/837#issuecomment-864899918
+import selectors
+selectors._PollLikeSelector.modify = selectors._BaseSelectorImpl.modify
+
+import json
+
+import aiopg
+from psycopg2.extras import NamedTupleCursor
+import sentry_sdk
+
+from overpass import ZOOM_DEFAULT
+
+
+tile_query = """
+    SELECT * from soundscape_tile(%(zoom)s, %(tile_x)s, %(tile_y)s)
+"""
+
+
+class PostgisClient:
+    """A drop-in replacement for OverpassClient that uses a PostGIS server.
+    The server is assumed to already be populated, including having the
+    soundscape_tile function installed.
+    """
+    def __init__(self, server):
+        self.server = server
+
+    @sentry_sdk.trace
+    async def query(self, x, y):
+        try:
+            async with aiopg.connect(self.server) as conn:
+                async with conn.cursor(cursor_factory=NamedTupleCursor) as cursor:
+                    response = await self._gentile_async(cursor, x, y)
+                    return response
+        except Exception as e:
+            print(e)
+            raise
+
+    # based on https://github.com/microsoft/soundscape/blob/main/svcs/data/gentiles.py
+    async def _gentile_async(self, cursor, x, y, zoom=ZOOM_DEFAULT):
+        await cursor.execute(tile_query, {'zoom': int(zoom), 'tile_x': x, 'tile_y': y})
+        value = await cursor.fetchall()
+        return {
+            'type': 'FeatureCollection',
+            'features': list(map(lambda x: x._asdict(), value))
+        }

app/server.py

@@ -1,17 +1,27 @@
 #!/usr/bin/env python3
 import json
+from urllib.parse import urlparse
+
 from aiohttp import web
-from overpass import ZOOM_DEFAULT, OverpassClient
 import sentry_sdk
 from sentry_sdk.integrations.aiohttp import AioHttpIntegration
 
+from overpass import ZOOM_DEFAULT, OverpassClient
+from postgis import PostgisClient
+
 import logging
 logger=logging.getLogger(__name__)
 
+# workaround for aiohttp on WIndows (https://stackoverflow.com/a/69195609)
+import sys, asyncio
+if sys.version_info >= (3, 8) and sys.platform.lower().startswith("win"):
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+
 # based on https://github.com/microsoft/soundscape/blob/main/svcs/data/gentiles.py
 @sentry_sdk.trace
-async def gentile_async(zoom, x, y, overpass_client):
-    response = await overpass_client.query(x, y)
+async def gentile_async(zoom, x, y, backend_client):
+    response = await backend_client.query(x, y)
     if response is None:
         return response
     return json.dumps(response, sort_keys=True)
@@ -26,7 +36,7 @@ async def tile_handler(request):
     x = int(request.match_info["x"])
     y = int(request.match_info["y"])
     try:
-        tile_data = await gentile_async(zoom, x, y, request.app["overpass_client"])
+        tile_data = await gentile_async(zoom, x, y, request.app["backend_client"])
         if tile_data == None:
             raise web.HTTPServiceUnavailable()
         else:
@@ -35,10 +45,21 @@ async def tile_handler(request):
         logger.error(f"request: {request.rel_url}")
 
 
+def backend_client(backend_url, user_agent, cache_dir, cache_days, cache_size):
+    """Determine which backend to use based on URL format."""
+    url_parts = urlparse(backend_url)
+    if url_parts.scheme in ('http', 'https'):
+       return OverpassClient(
+            backend_url, user_agent, cache_dir, cache_days, cache_size
+        )
+    elif url_parts.scheme in ('postgis', 'postgres'):
+        return PostgisClient(backend_url)
+    else:
+        raise ValueError("Unrecognized protocol %r" % url_parts.scheme)
 
 
 def run_server(
-    overpass_url,
+    backend_url,
     user_agent,
     cache_dir,
     cache_days,
@@ -55,12 +76,11 @@ def run_server(
         ],
     )
     sentry_sdk.set_tag(
-        "overpass_url", overpass_url
+        "backend_url", backend_url
     )  # Tag all requests for the lifecycle of the app with the overpass URL used
     app = web.Application()
-    app["overpass_client"] = OverpassClient(
-        overpass_url, user_agent, cache_dir, cache_days, cache_size
-    )
+    app["backend_client"] =backend_client(
+        backend_url, user_agent, cache_dir, cache_days, cache_size)
     app.add_routes(
         [
             web.get(r"/tiles/{zoom:\d+}/{x:\d+}/{y:\d+}.json", tile_handler),

app/tests.py

@@ -10,6 +10,8 @@
 
 from cache import CompressedJSONCache
 from overpass import OverpassClient, OverpassResponse
+from postgis import PostgisClient
+from server import backend_client
 
 
 class TestCompressedJSONCache:
@@ -221,3 +223,27 @@ async def test_intersections(self, x, y, overpass_client):
                         )
                     )
                 )
+
+
+class TestPostgisClient:
+    @pytest.mark.parametrize(
+        "url,expected_type",
+        [
+            ["https://overpass.kumi.systems/api/interpreter/", OverpassClient],
+            ["postgres://username:[email protected]:5432/osm/", PostgisClient],
+            ["ftp://example.com/", ValueError],
+        ],
+    )
+    def test_url_recognition(self, url, expected_type):
+        """We should get an OverpassClient, PostgisClient, or ValueError based on the URL."""
+        try:
+            return_value = backend_client(
+                url,
+                "Overscape/0.1",
+                cache_dir=Path("_test_cache"),
+                cache_days=7,
+                cache_size=1e5,
+            )
+        except Exception as exc:
+            return_value = exc
+        assert type(return_value) is expected_type

docker-compose.yml

@@ -29,7 +29,7 @@ services:
     build:
       dockerfile: Dockerfile
     environment:
-      - OVERPASS_URL=http://overpass/api/interpreter
+      - BACKEND_URL=http://overpass/api/interpreter
       - SENTRY_DSN=$SENTRY_DSN
       - SENTRY_TSR=$SENTRY_TSR
     ports:

requirements.txt

@@ -1,4 +1,5 @@
 aiohttp==3.8.4
+aiopg==1.4.0
 osm2geojson==0.2.3
 shapely==2.0.1
 sentry-sdk==1.25.1