fix(merge): merge conflice solved

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
   # Lint / autoformat: Python code
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.8.6"
+    rev: "v0.9.1"
     hooks:
       # Run the linter
       - id: ruff
@@ -21,7 +21,7 @@ repos:
 
   # Deps: ensure Python uv lockfile is up to date
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.5.14
+    rev: 0.5.18
     hooks:
       - id: uv-lock
         files: src/backend/pyproject.toml

contrib/just/start/Justfile

@@ -32,7 +32,7 @@ backend:
   cd {{justfile_directory()}}
   docker compose up -d api
 
-# Start backend API only
+# Start backend API without docker
 [no-cd]
 backend-no-docker:
   #!/usr/bin/env sh
@@ -43,11 +43,34 @@ backend-no-docker:
   OSM_SECRET_KEY="" ENCRYPTION_KEY="" \
     uv run uvicorn app.main:api --host 0.0.0.0 --port 8000
 
-# Start frontend UI only
+# Start frontend UI (also starts backend)
 [no-cd]
 frontend:
   docker compose up -d ui
 
+# Start frontend UI without docker, connected to staging
+[no-cd]
+frontend-dev:
+  #!/usr/bin/env sh
+
+  cd {{justfile_directory()}}/src/frontend
+
+  pnpm install
+  VITE_API_URL=https://api.stage.fmtm.hotosm.org \
+  pnpm run dev
+
+# Start mapper frontend UI without docker, connected to staging
+[no-cd]
+mapper-frontend-dev:
+  #!/usr/bin/env sh
+
+  cd {{justfile_directory()}}/src/mapper
+
+  pnpm install
+  VITE_API_URL=https://api.stage.fmtm.hotosm.org \
+  VITE_SYNC_URL=https://sync.stage.fmtm.hotosm.org \
+  pnpm run dev
+
 # Start FMTM without ODK Central
 [no-cd]
 without-central:

docs/decisions/0000-hotosm.md

@@ -0,0 +1,7 @@
+# Higher Level HOTOSM Decisions
+
+Many decisions have been made at an organizational level, affecting
+all tools that we develop.
+
+The decisions made in this project should not deviate much from the
+choices, [starting here](https://docs.hotosm.org/decisions/0003-react).

docs/decisions/0002-odk.md

@@ -0,0 +1,40 @@
+# Use ODK as the underlying survey data collection tool
+
+## Context and Problem Statement
+
+FMTM requires the collection of structured data from users in the field,
+using mobile phone-based data forms.
+
+To save re-inventing the wheel, many other organizations have addressed
+this common requirement, so we should incorporate their tooling into
+FMTM.
+
+## Considered Options
+
+- Kobo
+- ODK
+- Ona.io
+- Dimagi CommCare
+
+## Decision Outcome
+
+We chose ODK for two main reasons:
+
+- HOT staff have used ODK successfully for many field data collection campaigns
+  in the past, and can attest to it's suitableness.
+- ODK appears to be at the forefront of this technology, for both product
+  stability, and new feature innovation. Others mimic the same technology,
+  building wrappers around ODK's tools.
+
+### Consequences
+
+- Good, because we do not need to implement the form creation logic.
+- Good, because ODK uses simple technologies accessible to most users,
+  such as XLSX spreadsheets to build forms.
+- Good, because we do not need to implement the field data collection
+  tool. Instead we need to develop the coordination layer above.
+- Good, because ODK is extremely robust and battle tested. We can rely on it.
+- Bad, we now have two tools and two databases to manage.
+- Bad, it adds additional constraints to development, having to
+  manage data flows in both FMTM and ODK (for example, as local-first
+  approach to development becomes significantly harder).

docs/decisions/0003-flatgeobuf.md

@@ -0,0 +1,46 @@
+# Data extract files should be in flatgeobuf format
+
+## Context and Problem Statement
+
+When creating an FMTM project, we need a set of geometries to map. This
+has typically been called the 'data extract'.
+
+Either the user can upload their own geometries in GeoJSON format, or
+download geometries from OSM via a call to raw-data-api.
+
+The resulting data should be easily accessible on the backend for creation of
+a matching ODK Entities Dataset (used for the geometry selection in ODK), but
+also on the frontend for easy visualisation (ideally without an API call).
+
+A cloud-native format is most appropriate here, to reduce calls to the API,
+and outsource to object storage like S3.
+
+## Considered Options
+
+- GeoJSON
+- Flatgeobuf
+- Geoparquet
+- PMTiles (or other vector tiles)
+
+## Decision Outcome
+
+<https://github.com/hotosm/fmtm/pull/1047>
+
+At the time of implementation (Dec 2023), `flatgeobuf` was selected as the best
+candidate. The frontend JS implementation is excellent.
+
+- GeoJSON is not cloud native / BBOX accessible via a HTTP RANGE request.
+- GeoParquet support on the frontend was lacking.
+- PMTile usage was still quite nascent. Since then this has been identified
+  as another possible solution. However, creation would require additional libs
+  to be bundled in FMTM, and vector tile styling can be a pain.
+
+Flatgeobuf seems to be most applicable for this use case (a small amount of data).
+
+### Consequences
+
+- Good, because it's a simple format with excellent support in GIS tools (OGR).
+- Good, no styling is required. It can be determined by the web library / renderer.
+- Good, has built in geospatial index and accepts BBOX-based HTTP RANGE requests.
+- Bad, because flatgeobuf does not support compression, so files are marginally
+  larger.

docs/decisions/0004-web-forms.md

@@ -0,0 +1,42 @@
+# Implement ODK Web Forms to phase out ODK Collect
+
+## Context and Problem Statement
+
+ODK Collect is the mobile survey data collection portion of ODK.
+
+It currently only supports Android.
+
+In FMTM, the mapper is required to swap back-and-forth between the FMTM
+and ODK Collect applications, which is not an ideal experience.
+
+Users on iOS also wish to use the FMTM.
+
+## Considered Options
+
+- ODK Collect only
+- ODK Collect + GIC Collect (unofficial iOS port)
+- ODK Web Forms
+- Custom web implementation in FMTM
+
+## Decision Outcome
+
+Between 01-2025 and 05-2025, ODK Web forms will be integrated into FMTM.
+
+Again, we can extend from the excellent work done by ODK on the core
+xform-engine logic, then either use their Vue based component implementation
+in a Web Component wrapper, or write our own components wrapping this logic.
+
+GIC Collect is closed source and updated infrequently, so not an option.
+
+### Consequences
+
+- Good, because we have to maintain less code ourselves.
+- Good, because it broadens the potential user base of FMTM.
+- Good, because it simplifies the tech stack for FMTM, with no interfacing
+  to an Android application via deep linking / intents required.
+- Bad, because it's a risk with Web Forms being such a new project.
+- Bad, because it means we are entirely web-based. This has pros and cons,
+  but we do lose the benefit of very easy offline submission management
+  in ODK Collect (everything required is simply bundled in the app, with
+  no web dependencies). We are banking on good caching, intermittent
+  connectivity, and a sync layer to facilitate offline-first.

docs/decisions/README.md

@@ -1,6 +1,6 @@
 # Architectural Decisions
 
-A Markdown and Architectural Decision Records documenting the technical decision
+Markdown Architectural Decision Records documenting the technical decisions
 taken in this project.
 
 This process was started 29/10/2024, so does not necessarily capture all decisions

docs/dev/Frontend.md

@@ -30,19 +30,17 @@ For details on how to run the API first, please see:
 
 ## 2. Start the Frontend locally
 
-### 2A: Navigate to the frontend subdirectory
+To run the frontend locally, connected to the staging server as a backend:
 
-`cd src/frontend`
-
-### 2B: Install dependencies
-
-`npm install`
-
-### 2C. Run the project
+```bash
+just --unstable start frontend-dev
+```
 
-Run the frontend with hot-reloading: `npm run dev`
+The mapper frontend can be started with a similar command:
 
-The frontend should now be accessible at: `http://127.0.0.1:<PORT_NUMBER>`
+```bash
+just --unstable start mapper-frontend-dev
+```
 
 ## Frontend Tips
 

docs/manuals/mapping.md

@@ -131,6 +131,8 @@ In most cases we are submitting a survey about a feature that already exists.
   feature pre-selected in the survey (no need to open the ODK map!).
 - Complete the survey and submit.
 
+![IMG_20250109_160742](https://github.com/user-attachments/assets/bf350d1c-c80e-42ee-970b-ca71a3713a9f)
+
 !!! tip
 
       You may manually click 'Start Mapping' for a task area.
@@ -155,6 +157,7 @@ Sometimes the feature does not exist on the map yet!
 - Click on the map to create a new geometry.
 - ODK Collect will be opened automatically to fill out the survey
   data for the newly created feature.
+  ![IMG_20250109_160816](https://github.com/user-attachments/assets/98b70f5a-4db8-46cb-84ae-58bec07c82c1)
 
 !!! note
 
@@ -173,7 +176,9 @@ Sometimes the feature does not exist on the map yet!
   another feature for mapping.
 - You also also have to click the **Sync Status** button to see the feature
   turn green.
+
   ![sync-status-button](https://github.com/user-attachments/assets/38062aad-c8ea-4d47-a617-4be70dbfa20c)
+
 - Select another feature to map (repeat from step 5)!
 
 !!! note

scripts/1-environment/gen-env.sh

@@ -324,7 +324,7 @@ set_osm_credentials() {
 
     export OSM_CLIENT_ID=${OSM_CLIENT_ID}
     export OSM_CLIENT_SECRET=${OSM_CLIENT_SECRET}
-    secret_key=$(tr -dc 'a-zA-Z0-9' </dev/urandom | head -c 50)
+    secret_key=$(base64 </dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 50)
     export OSM_SECRET_KEY=${secret_key}
 }
 

src/Dockerfile.ui.debug

@@ -9,6 +9,6 @@ WORKDIR /app
 COPY --from=code ./package.json ./pnpm-lock.yaml ./
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
-RUN corepack enable && corepack use pnpm@9.3.0
+RUN corepack enable && corepack use pnpm@9.15.4
 RUN pnpm install
 ENTRYPOINT ["pnpm", "run", "dev"]

src/Dockerfile.ui.prod

@@ -5,7 +5,7 @@ ENV VITE_API_URL=${VITE_API_URL} \
     VITE_SYNC_URL=${VITE_SYNC_URL} \
     PNPM_HOME="/pnpm" \
     PATH="$PATH:/pnpm"
-RUN corepack enable && corepack use pnpm@9.3.0
+RUN corepack enable && corepack use pnpm@9.15.4
 WORKDIR /app
 
 

src/backend/app/central/central_crud.py

@@ -34,7 +34,7 @@
 
 from app.central import central_deps, central_schemas
 from app.config import settings
-from app.db.enums import EntityState, HTTPStatus
+from app.db.enums import DbGeomType, EntityState, HTTPStatus
 from app.db.models import DbXLSForm
 from app.db.postgis_utils import (
     geojson_to_javarosa_geom,
@@ -323,6 +323,7 @@ async def append_fields_to_user_xlsform(
     form_category: str = "buildings",
     additional_entities: list[str] = None,
     existing_id: str = None,
+    new_geom_type: DbGeomType = DbGeomType.POINT,
 ) -> tuple[str, BytesIO]:
     """Helper to return the intermediate XLSForm prior to convert."""
     log.debug("Appending mandatory FMTM fields to XLSForm")
@@ -331,6 +332,7 @@ async def append_fields_to_user_xlsform(
         form_category=form_category,
         additional_entities=additional_entities,
         existing_id=existing_id,
+        new_geom_type=new_geom_type,
     )
 
 
@@ -339,13 +341,15 @@ async def validate_and_update_user_xlsform(
     form_category: str = "buildings",
     additional_entities: list[str] = None,
     existing_id: str = None,
+    new_geom_type: DbGeomType = DbGeomType.POINT,
 ) -> BytesIO:
     """Wrapper to append mandatory fields and validate user uploaded XLSForm."""
     xform_id, updated_file_bytes = await append_fields_to_user_xlsform(
         xlsform,
         form_category=form_category,
         additional_entities=additional_entities,
         existing_id=existing_id,
+        new_geom_type=new_geom_type,
     )
 
     # Validate and return the form

src/backend/app/config.py

@@ -97,7 +97,7 @@ def otel_python_excluded_urls(self) -> Optional[str]:
         # NOTE we add ODK Central session auth endpoint here
         if self.ODK_CENTRAL_URL:
             os.environ["OTEL_PYTHON_REQUESTS_EXCLUDED_URLS"] = (
-                f"{endpoints}" f"{self.ODK_CENTRAL_URL}/v1/sessions"
+                f"{endpoints}{self.ODK_CENTRAL_URL}/v1/sessions"
             )
         return endpoints
 
@@ -315,7 +315,7 @@ def get_settings():
     if _settings.DEBUG:
         # Enable detailed Python async debugger
         os.environ["PYTHONASYNCIODEBUG"] = "1"
-        print("Loaded settings: " f"{_settings.model_dump()}")
+        print(f"Loaded settings: {_settings.model_dump()}")
     return _settings