3.9.0.2

core/form_mission_settings.py

@@ -15,6 +15,7 @@
 from git import Repo
 
 from core import models
+from core.form_validation_biochem import BIOCHEM_CODES
 from core.forms import NoWhiteSpaceCharField
 from dart.utils import load_svg
 from settingsdb import models as settings_models, utils as settings_utils
@@ -109,7 +110,25 @@ def __init__(self, *args, **kwargs):
                 button_row.fields.append(Column(Div(HTML(f'<span class="me-2">{region}</span>'), button,
                                                     css_class="btn btn-outline-secondary"), css_class="col-auto"))
 
-        multi_select_col.fields.append(Row(Column(HTML(f'<span class="text-secondary">{multi_select_help_text}</span>')), css_class="mb-2"))
+        multi_select_col.fields.append(Row(
+            Column(HTML(f'<span class="text-secondary">{multi_select_help_text}</span>')), css_class="mb-2")
+        )
+
+        # if there's a validation error for a missing mission descriptor, highlight this field
+        descriptor_field = Field('mission_descriptor')
+        start_date_field = Field('start_date')
+        end_date_field = Field('end_date')
+        if self.instance.pk:
+            database = self.instance._state.db
+            if models.Error.objects.using(database).filter(
+                    code=BIOCHEM_CODES.DESCRIPTOR_MISSING.value).exists():
+                descriptor_field.attrs['class'] = descriptor_field.attrs.get('class', "") + " bg-danger-subtle"
+
+            date_issues = [BIOCHEM_CODES.DATE_MISSING.value, BIOCHEM_CODES.DATE_BAD_VALUES.value]
+            # if there's an issue with the dates highlight the date fields
+            if models.Error.objects.using(database).filter(code__in=date_issues).exists():
+                start_date_field.attrs['class'] = start_date_field.attrs.get('class', "") + " bg-danger-subtle"
+                end_date_field.attrs['class'] = end_date_field.attrs.get('class', "") + " bg-danger-subtle"
 
         submit = Submit('submit', 'Submit')
         self.helper.layout = Layout(
@@ -118,10 +137,10 @@ def __init__(self, *args, **kwargs):
             ),
             Row(
                 Column(
-                    Field('start_date')
+                    start_date_field
                 ),
                 Column(
-                    Field('end_date')
+                    end_date_field
                 )
             ),
             Row(
@@ -154,7 +173,7 @@ def __init__(self, *args, **kwargs):
                         css_class="alert alert-info ms-1 me-1"
                     ),
                     Row(
-                        Column(Field('mission_descriptor')),
+                        Column(descriptor_field),
                         Column(Field('lead_scientist')),
                     ),
                     Row(

core/form_validation_biochem.py

@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup
+from enum import Enum
 from django.urls import reverse_lazy, path
 from django.http import HttpResponse
 from django.utils.translation import gettext as _
@@ -11,29 +12,77 @@
 logger_notifications = logging.getLogger('dart.user.biochem_validation')
 
 
+class BIOCHEM_CODES(Enum):
+    # 1 - 1000 Date codes
+    DATE_MISSING = 1  # use for when a date is missing
+    DATE_BAD_VALUES = 2  # use when a date is improperly formatted or outside an expected range
+    POSITION_MISSING = 50 # use when an event/bottle is missing a position
+    DESCRIPTOR_MISSING = 1001  # use for when the mission descriptor is missing
+
+
+def _validation_mission_descriptor(mission: core_models.Mission) -> [core_models.Error]:
+    logger_notifications.info(_("Validating Mission descriptor"))
+    descriptor_errors = []
+
+    if not mission.mission_descriptor:
+        err = core_models.Error(mission=mission, type=core_models.ErrorType.biochem,
+                                message=_("Mission descriptor doesn't exist"),
+                                code=BIOCHEM_CODES.DESCRIPTOR_MISSING.value)
+        descriptor_errors.append(err)
+
+    return descriptor_errors
+
+
 def _validate_mission_dates(mission: core_models.Mission) -> [core_models.Error]:
     logger_notifications.info(_("Validating Mission Dates"))
 
     date_errors = []
     if not mission.start_date:
-        err = core_models.Error(mission=mission, type=core_models.ErrorType.biochem, message=_("Missing start date"))
+        err = core_models.Error(mission=mission, type=core_models.ErrorType.biochem, message=_("Missing start date"),
+                                code=BIOCHEM_CODES.DATE_MISSING.value)
         date_errors.append(err)
 
     if not mission.end_date:
-        err = core_models.Error(mission=mission, type=core_models.ErrorType.biochem, message=_("Missing end date"))
+        err = core_models.Error(mission=mission, type=core_models.ErrorType.biochem, message=_("Missing end date"),
+                                code=BIOCHEM_CODES.DATE_MISSING.value)
         date_errors.append(err)
 
     if mission.start_date and mission.end_date and mission.end_date < mission.start_date:
         err = core_models.Error(mission=mission, type=core_models.ErrorType.biochem,
-                                message=_("End date comes before Start date"))
+                                message=_("End date comes before Start date"),
+                                code=BIOCHEM_CODES.DATE_BAD_VALUES.value)
         date_errors.append(err)
 
     return date_errors
 
 
+def _validate_bottles(mission) -> [core_models.Error]:
+    logger_notifications.info(_("Validating Bottle Dates"))
+
+    errors: [core_models.Error] = []
+    database = mission._state.db
+    bottles = core_models.Bottle.objects.using(database).filter(event__mission=mission)
+
+    bottle_count = len(bottles)
+    for index, bottle in enumerate(bottles):
+        logger_notifications.info(_("Validating Bottles") + " : %d/%d", (index+1), bottle_count)
+
+        if not bottle.latitude:
+            # the biochem validation script only checks start dates, times, and positions
+            if not bottle.event.start_location:
+                err = core_models.Error(mission=mission, type=core_models.ErrorType.biochem,
+                                        code=BIOCHEM_CODES.POSITION_MISSING.value,
+                                        message=_("Event is missing a position. Event ID : ")+str(bottle.event.event_id)
+                                        )
+                errors.append(err)
+    return errors
+
+
 def validate_mission(mission: core_models.Mission) -> [core_models.Error]:
     errors = []
+    errors += _validation_mission_descriptor(mission)
     errors += _validate_mission_dates(mission)
+    errors += _validate_bottles(mission)
 
     return errors
 
@@ -55,7 +104,8 @@ def run_biochem_validation(request, database, mission_id):
 
     # 2. Re-run validation
     mission = core_models.Mission.objects.using(database).get(id=mission_id)
-    validate_mission(mission)
+    errors = validate_mission(mission)
+    core_models.Error.objects.using(database).bulk_create(errors)
 
     response = HttpResponse()
     response['HX-Trigger'] = 'biochem_validation_update'
@@ -81,11 +131,22 @@ def get_validation_errors(request, database, mission_id):
     soup.append(ul := soup.new_tag('ul'))
     ul.attrs = {'class': 'list-group'}
 
+    # codes that should link the user back to the mission settings page to fix the issues.
+    settings_codes = [BIOCHEM_CODES.DESCRIPTOR_MISSING.value, BIOCHEM_CODES.DATE_MISSING.value,
+                      BIOCHEM_CODES.DATE_BAD_VALUES.value]
     for error in errors:
         ul.append(li := soup.new_tag('li'))
         li.attrs = {'class': 'list-group-item'}
         li.string = error.message
 
+        match error.code:
+            case code if code in settings_codes:
+                link = reverse_lazy('core:mission_edit', args=(database, mission_id))
+                li.string += " : "
+                li.append(a := soup.new_tag('a', href=link))
+                a.string = _("Mission Details")
+
+
     response = HttpResponse(soup)
     return response
 
@@ -94,4 +155,4 @@ def get_validation_errors(request, database, mission_id):
 database_urls = [
     path(f'{url_prefix}/biochem/validation/run/', run_biochem_validation, name="form_biochem_validation_run"),
     path(f'{url_prefix}/biochem/validation/', get_validation_errors, name="form_validation_get_validation_errors"),
-]
+]

core/migrations/0005_error_code_fileerror_code_validationerror_code.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2 on 2024-06-07 14:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0004_mission_dart_version'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='error',
+            name='code',
+            field=models.IntegerField(default=-1, verbose_name='Error code'),
+        ),
+        migrations.AddField(
+            model_name='fileerror',
+            name='code',
+            field=models.IntegerField(default=-1, verbose_name='Error code'),
+        ),
+        migrations.AddField(
+            model_name='validationerror',
+            name='code',
+            field=models.IntegerField(default=-1, verbose_name='Error code'),
+        ),
+    ]

core/models.py

@@ -598,6 +598,9 @@ class Meta:
     message = models.CharField(max_length=255, verbose_name=_("Message"))
     type = models.IntegerField(verbose_name=_("Error type"), default=0, choices=ErrorType.choices)
 
+    # The error code can be used to be more specific than an error type
+    code = models.IntegerField(verbose_name=_("Error code"), default=-1)
+
 
 # General errors we want to keep track of and notify the user about
 class Error(AbstractError):

core/parsers/andes.py

@@ -7,6 +7,7 @@
 from django.db.models import QuerySet
 from django.utils.translation import gettext_lazy as _
 
+import core.models
 from settingsdb.models import FileConfiguration
 
 from core import models as core_models
@@ -20,7 +21,7 @@ def get_or_create_file_config() -> QuerySet[FileConfiguration]:
     # These are all things the Elog parser requires, so we should probably figure out how to tackle them when reading
     # an Andes Report
     fields = [
-        ("lead_scientists", "chief_scientist", _("Label identifying the cheif scientists for the mission")),
+        ("lead_scientists", "chief_scientist", _("Label identifying the chief scientists for the mission")),
         ("platform", "name", _("Label identifying the ship name used for the mission")),
 
         ("instrument_name", "name", _("Label identifying an instrument name")),
@@ -142,7 +143,17 @@ def parse_events(mission: core_models.Mission, file_name: str, samples: list[dic
         for event in events:
             event_id = event[config.get(required_field='event_id').mapped_field]
             instrument_name = event[config.get(required_field='event_instrument_name').mapped_field]
-            instrument = instruments.get(name__iexact=instrument_name)
+            type_name = event[config.get(required_field='instrument_type').mapped_field]
+
+            if type_name.lower() == 'plankton net':
+                type_name = 'net'
+
+            type = core_models.InstrumentType.other
+            if core_models.InstrumentType.has_value(type_name):
+                type = core_models.InstrumentType.get(type_name)
+
+            instrument = instruments.get(name__iexact=instrument_name, type=type)
+
             sample_id = None
             end_sample_id = None
             if instrument.type == core_models.InstrumentType.ctd:

core/tests/TestFormValidationBiochem.py

@@ -94,11 +94,14 @@ def test_validate_missing_dates(self):
         self.assertEquals(errors[0].mission, bad_mission)
         self.assertEquals(errors[0].type, core_models.ErrorType.biochem)
         self.assertEquals(errors[0].message, _("Missing start date"))
+        self.assertEquals(errors[0].code, form_validation_biochem.BIOCHEM_CODES.DATE_MISSING.value)
 
         self.assertIsInstance(errors[1], core_models.Error)
         self.assertEquals(errors[1].mission, bad_mission)
         self.assertEquals(errors[1].type, core_models.ErrorType.biochem)
         self.assertEquals(errors[1].message, _("Missing end date"))
+        self.assertEquals(errors[0].code, form_validation_biochem.BIOCHEM_CODES.DATE_MISSING.value)
+
 
     @tag('form_validation_biochem_test_validate_bad_dates')
     def test_validate_bad_dates(self):
@@ -114,3 +117,48 @@ def test_validate_bad_dates(self):
         self.assertEquals(errors[0].mission, bad_mission)
         self.assertEquals(errors[0].type, core_models.ErrorType.biochem)
         self.assertEquals(errors[0].message, _("End date comes before Start date"))
+        self.assertEquals(errors[0].code, form_validation_biochem.BIOCHEM_CODES.DATE_BAD_VALUES.value)
+
+    @tag('form_validation_biochem_test_mission_descriptor', 'git_issue_144')
+    def test_mission_descriptor(self):
+        # provided a mission with no name (used as the mission descriptor) to the _validation_mission_descriptor
+        # function an error should be reported.
+
+        bad_mission = core_factory.MissionFactory()
+        errors: [core_models.Error] = form_validation_biochem._validation_mission_descriptor(bad_mission)
+
+        self.assertIsNotNone(errors)
+
+        self.assertIsInstance(errors[0], core_models.Error)
+        self.assertEquals(errors[0].mission, bad_mission)
+        self.assertEquals(errors[0].type, core_models.ErrorType.biochem)
+        self.assertEquals(errors[0].message, _("Mission descriptor doesn't exist"))
+        self.assertEquals(errors[0].code, form_validation_biochem.BIOCHEM_CODES.DESCRIPTOR_MISSING.value)
+
+    @tag('form_validation_biochem_test_mission_descriptor', 'git_issue_144')
+    def test_validate_mission_descriptor_mission(self):
+        # ensure the _validate_mission_descriptor function is called through the validation_mission function
+        bad_mission = core_factory.MissionFactory()
+        errors: [core_models.Error] = form_validation_biochem.validate_mission(bad_mission)
+
+        self.assertIsNotNone(errors)
+
+        self.assertIsInstance(errors[0], core_models.Error)
+        self.assertEquals(errors[0].mission, bad_mission)
+        self.assertEquals(errors[0].type, core_models.ErrorType.biochem)
+        self.assertEquals(errors[0].message, _("Mission descriptor doesn't exist"))
+        self.assertEquals(errors[0].code, form_validation_biochem.BIOCHEM_CODES.DESCRIPTOR_MISSING.value)
+
+    @tag('form_validation_biochem_test_bottle_position_fail', 'git_issue_147')
+    def test_bottle_date_no_dates_fail(self):
+        # given an event with no date and a series of bottles with no dates validation should return an error
+        event = core_factory.CTDEventFactoryBlank(mission=self.mission)
+        bottles = core_factory.BottleFactory.create_batch(10, event=event)
+
+        errors: [core_models.Error] = form_validation_biochem._validate_bottles(self.mission)
+
+        self.assertIsInstance(errors[0], core_models.Error)
+        self.assertEquals(errors[0].mission, self.mission)
+        self.assertEquals(errors[0].type, core_models.ErrorType.biochem)
+        self.assertEquals(errors[0].message, _("Event is missing a position. Event ID : ") + str(event.event_id))
+        self.assertEquals(errors[0].code, form_validation_biochem.BIOCHEM_CODES.POSITION_MISSING.value)

core/tests/TestHtmx.py

@@ -24,13 +24,14 @@ def setUp(self) -> None:
         self.url = reverse_lazy(self.upload_elog_url, args=('default', self.mission.pk,))
         self.client = Client()
 
+    @tag('utils_elog_upload_test_elog_uplaod_missing_mid')
     def test_elog_uplaod_missing_mid(self):
         logger.info("Running test_elog_uplaod_missing_mid")
 
         file_name = 'missing_mid_bad.log'
 
         with open(self.file_location+file_name, 'rb') as fp:
-            self.client.post(self.url, {'event': fp})
+            self.client.post(self.url, {'elog_event': fp})
 
         errors = self.mission.file_errors.all()
         self.assertTrue(errors.exists())
@@ -46,7 +47,7 @@ def test_elog_uplaod_missing_fields(self):
         file_name = 'bad.log'
 
         with open(os.path.join(self.file_location, file_name), 'rb') as fp:
-            self.client.post(self.url, {'event': fp})
+            self.client.post(self.url, {'elog_event': fp})
 
         errors = self.mission.file_errors.all()
         self.assertTrue(errors.exists())