Merge branch 'dynamic_building_outlines' into mads-dynamic_building_o…

…utlines

.env.template

@@ -5,6 +5,8 @@ DATA_DIR_MODEL_OUTPUT=U:/Research/FloodRiskResearch/DigitalTwin/stored_data/mode
 DATA_DIR_GEOSERVER=U:/Research/FloodRiskResearch/DigitalTwin/stored_data/geoserver
 FLOOD_MODEL_DIR=U:/Research/FloodRiskResearch/DigitalTwin/BG-Flood/BG_Flood_v0-9
 
+DEBUG_TRACEBACK=False
+
 POSTGRES_HOST=localhost
 POSTGRES_PORT=5431
 POSTGRES_DB=db

Dockerfile

@@ -33,7 +33,7 @@ USER root
 
 # Install dependencies
 RUN apt-get update \
- && apt-get install -y --no-install-recommends ca-certificates curl wget \
+ && apt-get install -y --no-install-recommends ca-certificates curl wget acl \
 # Install firefox from mozilla .deb repository, not snap package as is default for ubuntu (snap does not work for docker)
  && wget -q https://packages.mozilla.org/apt/repo-signing-key.gpg -O- | tee /etc/apt/keyrings/packages.mozilla.org.asc > /dev/null \
  && echo "deb [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" | tee -a /etc/apt/sources.list.d/mozilla.list > /dev/null \
@@ -55,6 +55,9 @@ Pin-Priority: 1000 \n\
 # Remove unused packages. Keep curl for health checking in docker-compose
  && apt-get purge -y ca-certificates wget
 
+# Create stored data dir inside image, in case it does not get mounted (such as when deploying on AWS)
+RUN mkdir /stored_data && setfacl -R -m u:nonroot:rwx /stored_data
+
 USER nonroot
 
 # Copy python virtual environment from build layer
@@ -90,6 +93,7 @@ EXPOSE 5001
 SHELL ["/bin/bash", "-c"]
 # Activate environment and run the health-checker in background and celery worker in foreground
 ENTRYPOINT source /venv/bin/activate && \
-           health-checker --listener 0.0.0.0:5001 --script "celery -A src.tasks inspect ping" --script-timeout 10 & \
+           health-checker --listener 0.0.0.0:5001 --log-level error --script-timeout 10 \
+             --script "celery -A src.tasks inspect ping"  & \
            source /venv/bin/activate && \
            celery -A src.tasks worker -P threads --loglevel=INFO

docker-compose.yml

@@ -25,7 +25,7 @@ services:
     build:
       context: .
       target: backend
-    image: lparkinson/backend-flood-resilience-dt:1.0
+    image: lparkinson/backend-flood-resilience-dt:1.1
     container_name: backend_digital_twin
     env_file:
       - .env
@@ -50,7 +50,7 @@ services:
     build:
       context: .
       target: celery_worker
-    image: lparkinson/celery-flood-resilience-dt:1.0
+    image: lparkinson/celery-flood-resilience-dt:1.1
     container_name: celery_worker_digital_twin
     restart: always
     env_file:
@@ -97,7 +97,7 @@ services:
 
   www:
     # Webserver for the website interface
-    image: lparkinson/www-flood-resilience-dt:1.0
+    image: lparkinson/www-flood-resilience-dt:1.1
     build:
       context: ./visualisation
     container_name: www_digital_twin

src/app.py

@@ -96,10 +96,16 @@ def get_status(task_id: str) -> Response:
     """
     task_result = result.AsyncResult(task_id, app=tasks.app)
     status = task_result.status
-    task_value = task_result.get() if status == states.SUCCESS else None
     http_status = OK
-    if status == states.FAILURE:
+    if status == states.SUCCESS:
+        task_value = task_result.get()
+    elif status == states.FAILURE:
         http_status = INTERNAL_SERVER_ERROR
+        is_debug_mode = get_env_variable("DEBUG_TRACEBACK", default=False, cast_to=bool)
+        task_value = task_result.traceback if is_debug_mode else None
+    else:
+        task_value = None
+
     return make_response(jsonify({
         "taskId": task_result.id,
         "taskStatus": status,

src/config.py

@@ -41,6 +41,8 @@ def get_env_variable(var_name: str,
     env_var = os.getenv(var_name, default)
     if not allow_empty and env_var in (None, ""):
         raise KeyError(f"Environment variable {var_name} not set, and allow_empty is False")
+    if type(env_var) == cast_to:
+        return env_var
     return _cast_str(env_var, cast_to)
 
 
@@ -69,11 +71,11 @@ def _cast_str(str_to_cast: str, cast_to: Type[T]) -> T:
     if cast_to == bool:
         # For bool we have the problem where bool("False") == True but we want this function to return False
         truth_values = {"true", "t", "1"}
-        false_values = {"false", "f", "0"}
-        if str_to_cast.lower() in truth_values:
-            return True
-        elif str_to_cast.lower() in false_values:
+        false_values = {"false", "f", "0", ""}
+        if str_to_cast is None or str_to_cast.lower() in false_values:
             return False
+        elif str_to_cast.lower() in truth_values:
+            return True
         raise ValueError(f"{str_to_cast} being casted to bool but is not in {truth_values} or {false_values}")
     # General case
     return cast_to(str_to_cast)

src/tasks.py

@@ -4,6 +4,7 @@
 """
 import json
 import logging
+import traceback
 from typing import List, NamedTuple
 
 import geopandas as gpd
@@ -33,8 +34,12 @@
 class OnFailureStateTask(app.Task):
     """Task that switches state to FAILURE if an exception occurs"""
 
-    def on_failure(self, _exc, _task_id, _args, _kwargs, _einfo):
-        self.update_state(state=states.FAILURE)
+    def on_failure(self, exc, _task_id, _args, _kwargs, _einfo):
+        self.update_state(state=states.FAILURE, meta={
+            "exc_type": type(exc).__name__,
+            "exc_message": traceback.format_exc().split('\n'),
+            "extra": None
+        })
 
 
 class DepthTimePlot(NamedTuple):