Bug fixes for Celery 4.x and draft code to report e-mailing

Dockerfile

@@ -127,6 +127,8 @@ RUN ln -s "${KPI_SRC_DIR}/docker/init.bash" /etc/my_init.d/10_init_kpi.bash && \
     ln -s "${KPI_SRC_DIR}/docker/run_uwsgi.bash" /etc/service/uwsgi/run && \
     mkdir -p /etc/service/celery && \
     ln -s "${KPI_SRC_DIR}/docker/run_celery.bash" /etc/service/celery/run && \
+    mkdir -p /etc/service/celery_beat && \
+    ln -s "${KPI_SRC_DIR}/docker/run_celery_beat.bash" /etc/service/celery_beat/run && \
     mkdir -p /etc/service/celery_sync_kobocat_xforms && \
     ln -s "${KPI_SRC_DIR}/docker/run_celery_sync_kobocat_xforms.bash" /etc/service/celery_sync_kobocat_xforms/run
 

docker/run_celery.bash

@@ -4,9 +4,8 @@ source /etc/profile
 
 # Run the main Celery worker (will not process `sync_kobocat_xforms` jobs).
 cd "${KPI_SRC_DIR}"
-exec celery worker -A kobo --beat --loglevel=info \
+exec celery worker -A kobo --loglevel=info \
     --hostname=main_worker@%h \
     --logfile=${KPI_LOGS_DIR}/celery.log \
     --pidfile=/tmp/celery.pid \
     --exclude-queues=sync_kobocat_xforms_queue
-    --scheduler django_celery_beat.schedulers:DatabaseScheduler

docker/run_celery_beat.bash

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+source /etc/profile
+
+# Run the main Celery worker (will not process `sync_kobocat_xforms` jobs).
+cd "${KPI_SRC_DIR}"
+exec celery beat -A kobo --loglevel=info \
+    --logfile=${KPI_LOGS_DIR}/celery_beat.log \
+    --pidfile=/tmp/celery_beat.pid \
+    --scheduler django_celery_beat.schedulers:DatabaseScheduler

docker/run_celery_sync_kobocat_xforms.bash

@@ -12,6 +12,4 @@ exec celery worker -A kobo --loglevel=info \
     --pidfile=/tmp/celery_sync_kobocat_xforms.pid \
     --queues=sync_kobocat_xforms_queue \
     --concurrency=1 \
-    --maxtasksperchild=1
-    # Watch out: this may be changed in 4.x to `--max-tasks-per-child` per
-    # http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-max-tasks-per-child
+    --max-memory-per-child=1

kobo/apps/hook/tasks.py

@@ -45,4 +45,31 @@ def retry_all_task(hook_logs):
         hook_log.retry()
         time.sleep(0.2)
 
-    return True
+    return True
+
+
+@shared_task
+def failures_report():
+    """
+    Sends emails to owners' assets to report the submissions which failed
+    to be sent to external endpoint by hooks
+    :return:
+    """
+    from django.utils import translation
+    from django.core.mail import EmailMessage
+    from django.template import Context, Template
+    from django.template.loader import get_template
+
+    #translation.activate('fr')
+    template = get_template("reports/failures_email_body.txt")
+    variables = {}
+    text = template.render(Context(**variables))
+
+    msg = EmailMessage("Test", text, "[email protected]", [owner.email])
+    try:
+        msg.send()
+    except Exception as e:
+        print("ERROR - {}".format(str(e)))
+        return False
+
+    return True

kobo/apps/hook/tests/test_api_hook.py

@@ -33,7 +33,7 @@ def setUp(self):
         }
         self.asset.deployment._mock_submission(submission)
         self.asset.save(create_version=False)
-        settings.CELERY_ALWAYS_EAGER = True
+        settings.CELERY_TASK_ALWAYS_EAGER = True
 
     def _create_hook(self, return_response_only=False):
         url = reverse("hook-list", kwargs={"parent_lookup_asset": self.asset.uid})

kobo/settings.py

@@ -381,6 +381,7 @@ def __init__(self, *args, **kwargs):
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#new-lowercase-settings
 # http://docs.celeryproject.org/en/4.0/whatsnew-4.0.html#step-2-update-your-configuration-with-the-new-setting-names
 
+CELERY_TIMEZONE = "UTC"
 
 if os.environ.get('SKIP_CELERY', 'False') == 'True':
     # helpful for certain debugging