fix(backend): TMS issues by using go-tilepacks and go-pmtiles (#2162)

* refactor(backend): remove basemapper.py mbtile/pmtile generator, use go binaries

* build: add go-tilepacks & go-pmtiles binaries to backend for basemap gen

compose.yaml

@@ -76,7 +76,6 @@ services:
     # tty: true
     volumes:
       - fmtm_logs:/opt/logs
-      - fmtm_tiles:/opt/tiles
       - ./src/backend/pyproject.toml:/opt/pyproject.toml:ro
       - ./src/backend/app:/opt/app
       - ./src/backend/tests:/opt/tests:ro

contrib/basemaps/Dockerfile

@@ -0,0 +1,52 @@
+FROM docker.io/alpine/curl:8.11.1 AS base
+
+
+# Download and verify tilepack binary
+# FROM base AS tilepack
+# ENV GO_TILEPACK_URL=https://github.com/tilezen/go-tilepacks/releases/download/v1.0.0-pre1/tilepack_1.0.0-pre1_linux_amd64.tar.gz \
+#     TILEPACK_SHA1SUM=1f235fd3da7f9c8a2710a1a8d44a27c2c98df939
+# RUN curl -fsSLO "$GO_TILEPACK_URL" \
+#     && tar -xvzf tilepack_1.0.0-pre1_linux_amd64.tar.gz \
+#     && echo "${TILEPACK_SHA1SUM}  tilepack" | sha1sum -c - \
+#     && chmod +x tilepack \
+#     && mv tilepack /tilepack
+# FIXME temp workaround using fork until PRs merged
+# - https://github.com/tilezen/go-tilepacks/pull/36
+# - https://github.com/tilezen/go-tilepacks/pull/38
+FROM base AS tilepack
+ENV GO_TILEPACK_URL=https://github.com/spwoodcock/go-tilepacks/releases/download/0.3.0/tilepack_0.3.0_linux_amd64.tar.gz \
+    TILEPACK_SHA1SUM=6b7269735e9fb431de3457c8e2b6aa6d3f3ee49e
+RUN curl -fsSLO "$GO_TILEPACK_URL" \
+    && tar -xvzf tilepack_0.3.0_linux_amd64.tar.gz \
+    && echo "${TILEPACK_SHA1SUM}  tilepack" | sha1sum -c - \
+    && chmod +x tilepack \
+    && mv tilepack /tilepack
+
+
+
+# Download and verify pmtiles binary
+FROM base AS pmtiles
+ENV GO_PMTILES_URL=https://github.com/protomaps/go-pmtiles/releases/download/v1.25.0/go-pmtiles_1.25.0_Linux_x86_64.tar.gz \
+    PMTILES_SHA1SUM=ed4795e24bfcccc4fd07a54dfc7926e15cc835de
+RUN curl -fsSLO "$GO_PMTILES_URL" \
+    && tar -xvzf go-pmtiles_1.25.0_Linux_x86_64.tar.gz \
+    && echo "${PMTILES_SHA1SUM}  pmtiles" | sha1sum -c - \
+    && chmod +x pmtiles \
+    && mv pmtiles /pmtiles
+
+
+# Add a non-root user to passwd file
+FROM base AS useradd
+RUN addgroup -g 1000 nonroot
+RUN adduser -D -u 1000 -G nonroot nonroot
+
+
+# Deploy the application binary into scratch image
+FROM scratch AS release
+WORKDIR /app
+COPY --from=useradd /etc/group /etc/group
+COPY --from=useradd /etc/passwd /etc/passwd
+COPY --from=tilepack /tilepack /usr/bin/tilepack
+COPY --from=pmtiles /pmtiles /usr/bin/pmtiles
+USER nonroot:nonroot
+ENTRYPOINT ["tilepack"]

contrib/basemaps/README.md

@@ -0,0 +1,14 @@
+# Basemap Generator Image
+
+This container image contains binaries for:
+
+- <https://github.com/tilezen/go-tilepacks>
+- <https://github.com/protomaps/go-pmtiles>
+
+With these, we can generate an mbtiles file from TMS, then also convert to PMTiles.
+
+## Usage
+
+```bash
+docker run --rm ghcr.io/hotosm/fmtm/basemap-generator:0.3.0 --help
+```

contrib/basemaps/build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+IMAGE_NAME=ghcr.io/hotosm/fmtm/basemap-generator:0.3.0
+
+echo "Building ${IMAGE_NAME}"
+docker build . --tag "${IMAGE_NAME}"
+
+if [[ -n "$PUSH_IMG" ]]; then
+    docker push "${IMAGE_NAME}"
+fi

deploy/compose.development.yaml

@@ -83,7 +83,6 @@ services:
     image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}"
     volumes:
       - fmtm_logs:/opt/logs
-      - fmtm_tiles:/opt/tiles
     depends_on:
       fmtm-db:
         condition: service_healthy

deploy/compose.main.yaml

@@ -73,7 +73,6 @@ services:
     image: "ghcr.io/hotosm/fmtm/backend:main"
     volumes:
       - fmtm_logs:/opt/logs
-      - fmtm_tiles:/opt/tiles
     depends_on:
       fmtm-db:
         condition: service_healthy

src/backend/Dockerfile

@@ -19,6 +19,7 @@ ARG UV_IMG_TAG=0.5.2
 ARG MINIO_TAG=${MINIO_TAG:-RELEASE.2025-01-20T14-49-07Z}
 FROM ghcr.io/astral-sh/uv:${UV_IMG_TAG} AS uv
 FROM docker.io/minio/minio:${MINIO_TAG} AS minio
+FROM ghcr.io/hotosm/fmtm/basemap-generator:0.3.0 AS basemap-bins
 
 
 # Includes all labels and timezone info to extend from
@@ -121,6 +122,9 @@ RUN apt-get update --quiet \
     && rm -rf /var/lib/apt/lists/*
 # Copy minio mc client
 COPY --from=minio /usr/bin/mc /usr/local/bin/
+# Copy basemap generation binaries
+COPY --from=basemap-bins /usr/bin/tilepack /usr/local/bin/
+COPY --from=basemap-bins /usr/bin/pmtiles /usr/local/bin/
 COPY *-entrypoint.sh /
 ENTRYPOINT ["/app-entrypoint.sh"]
 # Copy Python deps from build to runtime
@@ -131,12 +135,11 @@ COPY app/ /opt/app/
 COPY migrations/ /opt/migrations/
 # Add non-root user, permissions
 RUN useradd -u 1001 -m -c "fmtm account" -d /home/appuser -s /bin/false appuser \
-    && mkdir -p /opt/logs /opt/tiles \
+    && mkdir -p /opt/logs \
     && chown -R appuser:appuser /opt /home/appuser \
     && chmod +x /app-entrypoint.sh /migrate-entrypoint.sh /backup-entrypoint.sh
-# Add volumes for persistence
+# Add log volume for persistence
 VOLUME /opt/logs
-VOLUME /opt/tiles
 # Change to non-root user
 USER appuser
 

src/backend/app/projects/project_crud.py

@@ -18,7 +18,7 @@
 """Logic for FMTM project routes."""
 
 import json
-import shutil
+import subprocess
 import uuid
 from io import BytesIO
 from pathlib import Path
@@ -32,7 +32,6 @@
 from asgiref.sync import async_to_sync
 from fastapi import HTTPException, Request
 from loguru import logger as log
-from osm_fieldwork.basemapper import create_basemap_file
 from osm_login_python.core import Auth
 from osm_rawdata.postgres import PostgresClient
 from psycopg import Connection
@@ -56,8 +55,6 @@
 from app.projects import project_deps, project_schemas
 from app.s3 import add_file_to_bucket, add_obj_to_bucket
 
-TILESDIR = "/opt/tiles"
-
 
 async def get_projects_featcol(
     db: Connection,
@@ -727,15 +724,49 @@ def generate_project_basemap(
         tms (str, optional): Default None. Custom TMS provider URL.
     """
     new_basemap = None
+    mbtiles_file = ""
+    final_tile_file = ""
 
     # TODO update this for user input or automatic
     # maxzoom can be determined from OAM: https://tiles.openaerialmap.org/663
     # c76196049ef00013b8494/0/663c76196049ef00013b8495
-    # TODO xy should also be user configurable
+    # TODO should inverted_y be user configurable?
+
     # NOTE mbtile max supported zoom level is 22 (in GDAL at least)
-    zooms = "12-22" if tms else "12-19"
-    tiles_dir = f"{TILESDIR}/{project_id}"
-    outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}"
+    if tms:
+        zooms = "12-22"
+    # While typically satellite imagery TMS only goes to zoom 19
+    else:
+        zooms = "12-19"
+
+    mbtiles_file = Path(f"/tmp/{project_id}_{source}tiles.mbtiles")
+
+    # Set URL based on source (previously in osm-fieldwork)
+    if source == "esri":
+        # ESRI uses inverted zyx convention
+        # the ordering is extracted automatically from the URL, else use
+        # -inverted-y param
+        tms_url = "http://services.arcgisonline.com/arcgis/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}.png"
+        tile_format = "png"
+    elif source == "bing":
+        # FIXME this probably doesn't work
+        tms_url = "http://ecn.t0.tiles.virtualearth.net/tiles/h{z}/{x}/{y}.jpg?g=129&mkt=en&stl=H"
+        tile_format = "jpg"
+    elif source == "google":
+        tms_url = "https://mt0.google.com/vt?lyrs=s&x={x}&y={y}&z={z}"
+        tile_format = "jpg"
+    elif source == "custom" and tms:
+        tms_url = tms
+        if not (tile_format := Path(tms_url).suffix.lstrip(".")):
+            # Default to png if suffix not included in URL
+            tile_format = "png"
+    else:
+        raise ValueError("Must select a source from: esri,bing,google,custom")
+
+    # Invert zxy --> zyx for OAM provider
+    # inverted_y = True if tms and "openaerialmap" in tms else False
+    # NOTE the xy ordering is determined from the URL placeholders, by tilepack!
+    inverted_y = False
 
     # NOTE here we put the connection in autocommit mode to ensure we get
     # background task db entries if there is an exception.
@@ -757,36 +788,65 @@ def generate_project_basemap(
 
         min_lon, min_lat, max_lon, max_lat = new_basemap.bbox
 
-        # Overwrite source with OAM provider
-        if tms and "openaerialmap" in tms:
-            # NOTE the 'xy' param is set automatically by source=oam
-            source = "oam"
-
+        tilepack_cmd = [
+            # "prlimit", f"--as={500 * 1000}", "--",
+            "tilepack",
+            "-dsn",
+            f"{str(mbtiles_file)}",
+            "-url-template",
+            f"{tms_url}",
+            # tilepack requires format: south,west,north,east
+            "-bounds",
+            f"{min_lat},{min_lon},{max_lat},{max_lon}",
+            "-zooms",
+            f"{zooms}",
+            "-output-mode",
+            "mbtiles",  # options: mbtiles or disk
+            "-mbtiles-format",
+            f"{tile_format}",
+            "-ensure-gzip=false",  # gzip is only used for pbf vector tiles
+            "-tileset-name",
+            f"fmtm_{project_id}_{source}tiles",
+        ]
+        # Add '-inverted-y' only if needed
+        if inverted_y:
+            tilepack_cmd.append("-inverted-y")
         log.debug(
-            "Creating basemap with params: "
-            f"boundary={min_lon},{min_lat},{max_lon},{max_lat} | "
-            f"outfile={outfile} | "
-            f"zooms={zooms} | "
-            f"outdir={tiles_dir} | "
-            f"source={source} | "
-            f"tms={tms}"
+            "Creating basemap mbtiles using tilepack with command: "
+            f"{' '.join(tilepack_cmd)}"
         )
-
-        create_basemap_file(
-            boundary=f"{min_lon},{min_lat},{max_lon},{max_lat}",
-            outfile=outfile,
-            zooms=zooms,
-            outdir=tiles_dir,
-            source=source,
-            tms=tms,
+        subprocess.run(tilepack_cmd, check=True)
+        log.info(
+            f"MBTile basemap created for project ID {project_id}: {str(mbtiles_file)}"
         )
-        log.info(f"Basemap created for project ID {project_id}: {outfile}")
+        # write to another var so we upload either mbtiles OR pmtiles override below
+        final_tile_file = str(mbtiles_file)
+
+        if output_format == "pmtiles":
+            pmtiles_file = mbtiles_file.with_suffix(".pmtiles")
+            pmtile_command = [
+                # "prlimit", f"--as={500 * 1000}", "--",
+                "pmtiles",
+                "convert",
+                f"{str(mbtiles_file)}",
+                f"{str(pmtiles_file)}",
+            ]
+            log.debug(
+                "Converting mbtiles --> pmtiles file with command: "
+                f"{' '.join(pmtile_command)}"
+            )
+            subprocess.run(pmtile_command, check=True)
+            log.info(
+                f"PMTile basemap created for project ID {project_id}: "
+                f"{str(pmtiles_file)}"
+            )
+            final_tile_file = str(pmtiles_file)
 
         # Generate S3 urls
         # We parse as BasemapOut to calculated computed fields (format, mimetype)
         basemap_out = project_schemas.BasemapOut(
             **new_basemap.model_dump(exclude=["url"]),
-            url=outfile,
+            url=final_tile_file,
         )
         basemap_s3_path = (
             f"{org_id}/{project_id}/basemaps/{basemap_out.id}.{basemap_out.format}"
@@ -795,7 +855,7 @@ def generate_project_basemap(
         add_file_to_bucket(
             settings.S3_BUCKET_NAME,
             basemap_s3_path,
-            outfile,
+            final_tile_file,
             content_type=basemap_out.mimetype,
         )
         basemap_external_s3_url = (
@@ -845,9 +905,9 @@ def generate_project_basemap(
             ),
         )
     finally:
-        log.info(f"Cleaned up tiles directory: {tiles_dir}")
-        Path(outfile).unlink(missing_ok=True)
-        shutil.rmtree(tiles_dir)
+        Path(mbtiles_file).unlink(missing_ok=True)
+        Path(final_tile_file).unlink(missing_ok=True)
+        log.debug("Cleaning up tile archives on disk")
 
 
 # async def convert_geojson_to_osm(geojson_file: str):