Benchmark heatmap algorithm and improvements

backend/diesel.toml

@@ -2,5 +2,5 @@
 file = "src/schema.rs"
 patch_file = "src/schema.patch"
 
-filter = { except_tables = ["spatial_ref_sys"] }
+filter = { except_tables = ["spatial_ref_sys", "relations_summary"] }
 import_types = ["postgis_diesel::sql_types::Geography", "diesel::sql_types::*"]

benchmarks/backend/heatmap_sql/.env.db

@@ -0,0 +1,5 @@
+POSTGRES_USER=permaplant
+POSTGRES_DB=permaplant
+POSTGRES_PASSWORD=permaplant
+PGPASSWORD=permaplant
+DATABASE_URL=postgres://permaplant:permaplant@db/permaplant

benchmarks/backend/heatmap_sql/Dockerfile.pg

@@ -0,0 +1,10 @@
+FROM postgis/postgis:13-3.1
+
+# Install plprofiler extension
+RUN apt-get update && \
+    apt-get install -y postgresql-13-plprofiler && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Expose the PostgreSQL port
+EXPOSE 5432

benchmarks/backend/heatmap_sql/Dockerfile.pga

@@ -0,0 +1,5 @@
+# This Dockerfile is used inside the docker-compose.yml as development environment
+FROM dpage/pgadmin4:7.3
+
+COPY ./servers.json ./servers.json
+COPY ./pgpass ../pgpass

benchmarks/backend/heatmap_sql/docker-compose.yml

@@ -0,0 +1,44 @@
+version: "3"
+
+services:
+  benchmark-postgis:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.pg
+    # image: postgis/postgis:13-3.1
+    container_name: benchmark-postgis
+    env_file:
+      - .env.db
+    ports:
+      - "5432:5432"
+    networks:
+      - permaplant-bechmark
+    restart: unless-stopped
+    volumes:
+      - postgis-data-benchmark:/var/lib/postgresql/data
+
+  bechmark-pgadmin:
+    build:
+      context: .
+      dockerfile: Dockerfile.pga
+    image: dpage/pgadmin4
+    restart: unless-stopped
+    volumes:
+      - pgadmin-data-benchmark:/var/lib/pgadmin
+    environment:
+      PGADMIN_DEFAULT_EMAIL: [email protected]
+      PGADMIN_DEFAULT_PASSWORD: permaplant
+    networks:
+      - permaplant-bechmark
+    ports:
+      - "5050:80"
+
+volumes:
+  pgadmin-data-benchmark:
+    external: false
+  postgis-data-benchmark:
+    external: false
+
+networks:
+  permaplant-bechmark:
+    external: false

benchmarks/backend/heatmap_sql/generate_data_to_insert.py

@@ -0,0 +1,59 @@
+import sys
+
+insert_sql = (
+    "DELETE FROM public.shadings WHERE layer_id = 3; \n"
+    "INSERT INTO public.shadings (id, layer_id, geometry, shade,  add_date, remove_date) VALUES \n"
+)
+
+
+if len(sys.argv) != 2:
+    print("Usage: python generate_data_to_insert.py NUM_SHADINGS")
+    print("NUM_SHADINGS must be 10, 100 or 1000")
+    exit(1)
+
+if int(sys.argv[1]) not in [10, 100, 1000]:
+    print("Usage: python generate_data_to_insert.py NUM_SHADINGS")
+    print("NUM_SHADINGS must be 10, 100 or 1000")
+    exit(1)
+
+NUM_SHADINGS = int(sys.argv[1])
+MAX_Y = MAX_X = 10000  # should be lower than large_map.sql bounds
+SHADE_WIDTH = 100
+STEP_SIZE = 150
+
+SHADING_TYPES = [
+    "light shade",
+    "partial shade",
+    "permanent shade",
+    "permanent deep shade",
+]
+
+
+shade_x = 0
+shade_y = 0
+
+assert STEP_SIZE > SHADE_WIDTH
+
+for i in range(NUM_SHADINGS):
+    shade_id = f"e6622aab-dc5d-4865-89d2-{i: 012d}"
+    shading_type = SHADING_TYPES[i % len(SHADING_TYPES)]
+
+    polygon = (
+        f"POLYGON(({shade_x} {shade_y}, {shade_x + SHADE_WIDTH} {shade_y}, "
+        f"{shade_x + SHADE_WIDTH} {shade_y + SHADE_WIDTH}, {shade_x} {shade_y + SHADE_WIDTH}, {shade_x} {shade_y}))"
+    )
+
+    insert_sql += f"('{shade_id}', 3, ST_GeomFromText('{polygon}'), '{shading_type}', NULL, NULL), \n"
+
+    shade_x += STEP_SIZE
+
+    if shade_x + SHADE_WIDTH >= MAX_X:
+        shade_x = 0
+        shade_y += STEP_SIZE
+
+    if shade_y + SHADE_WIDTH >= MAX_Y:
+        print("Error: Not enough space to place all shadings")
+        exit(1)
+
+
+print(insert_sql.rstrip(",\n") + ";")

benchmarks/backend/heatmap_sql/get_statistics.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env nu
+
+cd backend
+
+grep ' UTC \[[0-9]*\] ' postgis.log | lines | split column -r '\[|\]' | group-by column2 | values | each {|pid| $pid | each {|el| $"($el.column1)[($el.column2)]($el.column3)" } } | flatten | save -f postgis_parsed.log
+
+print "SQL queries:"
+grep 'LOG:  execute s.*: SELECT \* FROM calculate_heatmap' postgis_parsed.log -A 2
+    | grep 'LOG:  duration: .* ms$'
+    | awk 'NR == 1 {count=1; sum=$7; min=$7; max=$7}
+           NR > 1  {count++; sum+=$7; if ($7<0+min) min=$7; if ($7>0+max) max=$7}
+           END {print "total:", count, "\nmin:", min, "ms\navg:", sum/count, "ms\nmax:", max, "ms"}'
+
+print ""
+
+print "Requests:"
+grep 'Total: connections ' httperf.log | awk '{print "total:", $3}'
+grep 'Connection time \[ms\]: min' httperf.log | awk '{print "min:", $5, "ms\navg:", $7, "ms\nmax:", $9, "ms"}'