Merge pull request #637 from Terralego/mail_ending_refresh

Mail report when source is refreshed

project/celery.py

@@ -12,7 +12,7 @@
 BROKER_URL = f'redis://{os.getenv("REDIS_HOST", "redis")}:{os.getenv("REDIS_PORT", "6379")}/{os.getenv("REDIS_DB", "0")}'
 
 app.conf.update(
-    enable_utc=True,
+    timezone=settings.TIME_ZONE,
     accept_content=["json"],
     broker_url=BROKER_URL,
     task_serializer="json",

project/geosource/elasticsearch/index.py

@@ -75,7 +75,4 @@ def create_index(self):
         # Create query body with mapping
         body = {"mappings": {"properties": field_conf}}
 
-        try:
-            self.client.indices.create(index=self.layer.name, body=body)
-        except Exception:
-            logger.exception("ES index for layer {layer.name} can't be created")
+        self.client.indices.create(index=self.layer.name, body=body)

project/geosource/models.py

@@ -171,16 +171,18 @@ def refresh_data(self):
             es_index = LayerESIndex(layer)
             es_index.index()
             response = self._refresh_data(es_index)
+            self.status = self.Status.DONE.value
             return response
-        except SourceException as exc:
+
+        except Exception as exc:
             self.report.status = self.report.Status.ERROR.value
-            self.report.message = exc.message
+            self.report.message = str(exc)
             self.report.ended = timezone.now()
             self.report.save(update_fields=["status", "message", "ended"])
+            self.status = self.Status.DONE
 
         finally:
             self.last_refresh = timezone.now()
-            self.status = self.Status.DONE.value
             self.save()
             refresh_data_done.send_robust(
                 sender=self.__class__,

project/geosource/periodics.py

@@ -8,7 +8,6 @@
 def auto_refresh_source():
     countdown = 0
     for source in Source.objects.exclude(status=Source.Status.PENDING):
-        logger.info(f"Is refresh for {source}<{source.id}> needed?")
         if source.should_refresh():
             logger.info(f"Schedule refresh for source {source}<{source.id}>...")
             # Delay execution by some minutes to avoid struggling

project/geosource/tasks.py

@@ -2,8 +2,14 @@
 
 from celery import shared_task, states
 from celery.exceptions import Ignore
+from constance import config
 from django.apps import apps
+from django.core.mail import send_mail
+from django.template.loader import get_template
 from django.utils import timezone
+from django.utils.translation import gettext as _
+
+from project.visu.utils import get_logo_url
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +47,6 @@ def run_model_object_method(self, app, model, pk, method, success_state=states.S
 
     try:
         obj = Model.objects.get(pk=pk)
-        # raise Exception("not okkkkkkk")
         logger.info(f"Call method {method} on {obj}")
         state = {"action": method, **getattr(obj, method)()}
         logger.info(f"Method {method} on {obj} ended")
@@ -67,6 +72,41 @@ def run_model_object_method(self, app, model, pk, method, success_state=states.S
         set_failure_state(self, method, message, obj)
         logger.error(e, exc_info=True)
 
+    if config.INSTANCE_EMAIL_SOURCE_REFRESH_RECIPIENTS:
+        obj.refresh_from_db()
+        mail_level = config.INSTANCE_EMAIL_SOURCE_REFRESH_LEVEL
+        if (
+            mail_level == "success"
+            or (mail_level == "warning" and obj.report.status in (1, 2))
+            or (mail_level == "error" and obj.report.status == 1)
+        ):
+            emails = config.INSTANCE_EMAIL_SOURCE_REFRESH_RECIPIENTS.split(",")
+            logo_url = f"{config.INSTANCE_EMAIL_MEDIA_BASE_URL}{get_logo_url()}"
+            context = {
+                "title": config.INSTANCE_TITLE,
+                "obj": obj,
+                "logo_url": logo_url,
+            }
+            txt_template = get_template("emails/source_refresh/email.txt")
+            txt_message = txt_template.render(context=context)
+            html_template = get_template("emails/source_refresh/email.html")
+            html_message = html_template.render(context)
+            send_mail(
+                _(
+                    "%(title)s : Data source %(obj)s refresh ended with state %(success_state)s"
+                )
+                % {
+                    "title": config.INSTANCE_TITLE,
+                    "obj": obj,
+                    "success_state": obj.report.get_status_display(),
+                },
+                txt_message,
+                config.INSTANCE_EMAIL_FROM,
+                recipient_list=emails,
+                html_message=html_message,
+                fail_silently=True,
+            )
+
     raise Ignore()
 
 

project/geosource/tests/test_models.py

@@ -576,6 +576,7 @@ def test_partial_refresh_trigger_warning(
         )
         self.source._get_records = mock.MagicMock(return_value=mocked_rows)
         self.source.refresh_data()
+        self.source.refresh_from_db()
         self.assertEqual(
             self.source.report.status,
             SourceReporting.Status.WARNING.value,

project/geosource/tests/test_tasks.py

@@ -2,6 +2,7 @@
 from datetime import datetime
 from unittest import mock
 
+from constance.test import override_config
 from django.contrib.auth.models import Group
 from django.test import TestCase
 from geostore.models import Feature, Layer
@@ -25,6 +26,7 @@ def setUpTestData(cls):
         )
         cls.element.groups.add(cls.group)
 
+    @override_config(INSTANCE_EMAIL_SOURCE_REFRESH_RECIPIENTS="[email protected]")
     def test_task_refresh_data_method(
         self, mocked_index_feature, mocked_es_delete, mocked_es_create
     ):

project/locales/en/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-02-02 09:32+0000\n"
+"POT-Creation-Date: 2024-06-19 14:05+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <[email protected]>\n"
@@ -207,6 +207,11 @@ msgstr ""
 msgid "Incorrect value for coordinates field"
 msgstr ""
 
+#, python-format
+msgid ""
+"%(title)s : Data source %(obj)s refresh ended with state %(success_state)s"
+msgstr ""
+
 msgid "Instance title"
 msgstr ""
 
@@ -264,6 +269,22 @@ msgstr ""
 msgid "Content of info tab in frontend"
 msgstr ""
 
+msgid "Email addresses to send refresh done reports. (comma separated values)"
+msgstr ""
+
+msgid "Email address used as sender for emails sent."
+msgstr ""
+
+msgid ""
+"Minimum report status level to send email to refresh recipients. (success, "
+"warning, error)"
+msgstr ""
+
+msgid ""
+"Base url to prefix media files in HTML emails. (ex: https://terravisu.org/"
+"static_dj/)"
+msgstr ""
+
 msgid "Activate MapboxDraw Control to measure distance on the map"
 msgstr ""
 
@@ -300,6 +321,9 @@ msgstr ""
 msgid "General Options"
 msgstr ""
 
+msgid "Emails"
+msgstr ""
+
 msgid "Map max BBOX"
 msgstr ""
 
@@ -315,6 +339,88 @@ msgstr ""
 msgid "Other options"
 msgstr ""
 
+#, python-format
+msgid ""
+"\n"
+"                    %(title)s: Error when refreshing data source "
+"<b>\"%(source_name)s\"</b>\n"
+"                "
+msgstr ""
+
+#, python-format
+msgid ""
+"\n"
+"                %(title)s: Data source <b>\"%(source_name)s\"</b> refreshed\n"
+"            "
+msgstr ""
+
+#, python-format
+msgid ""
+"\n"
+"                    Hi,<br><br>\n"
+"                    Data source \"%(source_name)s\" refresh ended on "
+"<b>%(refresh_datetime)s</b>.<br>\n"
+"                "
+msgstr ""
+
+msgid "Objects"
+msgstr ""
+
+msgid "Total"
+msgstr ""
+
+msgid "Start"
+msgstr ""
+
+msgid "End"
+msgstr ""
+
+msgid "Adding"
+msgstr ""
+
+msgid "Updates"
+msgstr ""
+
+msgid "Deletion"
+msgstr ""
+
+#, python-format
+msgid ""
+"\n"
+"Hi,\n"
+"\n"
+"Data source \"%(source_name)s\" refresh ended on %(refresh_datetime)s.\n"
+"\n"
+"Started at %(refresh_start)s\n"
+"Ended at %(refresh_ended)s\n"
+"\n"
+"%(total)s total features\n"
+"\n"
+"with\n"
+"\n"
+"%(added_lines)s added features\n"
+"%(modified_lines)s modified features\n"
+"%(deleted_lines)s deleted features\n"
+"\n"
+msgstr ""
+
+#, python-format
+msgid ""
+"\n"
+"Its status is: %(status)s.\n"
+msgstr ""
+
+#, python-format
+msgid ""
+"\n"
+"No problem detected.\n"
+"\n"
+"Regards,\n"
+"\n"
+"The %(instance_title)s team\n"
+"\n"
+msgstr ""
+
 msgid "Layer identifier"
 msgstr ""