From 9ccb4c71fd4360fdf6c2f987e3dbfe953a0c3034 Mon Sep 17 00:00:00 2001 From: UpsonP Date: Wed, 10 Jul 2024 10:29:21 -0300 Subject: [PATCH] Fixed issue with missing sample id in btl file #173 --- core/parsers/SampleParser.py | 12 +++++++++++- core/parsers/ctd.py | 35 +++++++++++++++++++++++++++++------ core/tests/TestAndesParser.py | 1 + core/views_mission_sample.py | 5 ++++- start_dart.bat | 2 +- update.bat | 2 +- 6 files changed, 47 insertions(+), 10 deletions(-) diff --git a/core/parsers/SampleParser.py b/core/parsers/SampleParser.py index 17fbbad..b0a4cc2 100644 --- a/core/parsers/SampleParser.py +++ b/core/parsers/SampleParser.py @@ -262,7 +262,17 @@ def parse_data_frame(mission: core_models.Mission, sample_config: settings_model value = row[value_field] if sample_id not in bottle_keys: - message = f"Could not find bottle matching id {sample_id} in file {file_name}" + message = f"Could not find bottle matching id {sample_id} in file {file_name} \n" + event = mission.events.filter(sample_id__lte=sample_id, end_sample_id__gte=sample_id) + if event: + event = event.first() + message += (_("Possible Event") + f" #{event.event_id} " + _("Bottle IDs") + + f" {event.sample_id} - {event.end_sample_id}") + message += _("\nEvent Comments : ") + for action in event.actions.all(): + if action.comment: + message += f"\n{action.get_type_display()} : {action.comment}" + error = core_models.FileError(mission=mission, file_name=file_name, line=sample_id, message=message, type=core_models.ErrorType.sample) errors.append(error) diff --git a/core/parsers/ctd.py b/core/parsers/ctd.py index 42987cf..77558b6 100644 --- a/core/parsers/ctd.py +++ b/core/parsers/ctd.py @@ -337,10 +337,15 @@ def process_data(event: core_models.Event, data_frame: pandas.DataFrame, column_ update_discrete_samples: [core_models.DiscreteSampleValue] = [] # validate and remove bottles that don't exist if data_frame_avg.shape[0] > (event.total_samples + 1): - message = _('Event contained more than the expected number of bottles. Additional bottles will be dropped. ') - message += _("Event") + f" #{event.event_id} " - message += _("Expected") + f"[{(event.total_samples+1)}] " - message += _("Found") + f"[{data_frame_avg.shape[0]}]" + message = _('Event contained more than the expected number of bottles. Additional bottles will be dropped. \n') + message += _("Event") + f" #{event.event_id} \n" + message += _("Expected") + f"[{(event.total_samples+1)}] \n" + message += _("Found") + f"[{data_frame_avg.shape[0]}]\n" + message += _("\nEvent Comments : ") + for action in event.actions.all(): + if action.comment: + message += f"\n{action.get_type_display()} : {action.comment}" + logger.warning(message) core_models.FileError(mission=mission, file_name=file_name, line=skipped_rows, message=message, @@ -373,11 +378,29 @@ def process_data(event: core_models.Event, data_frame: pandas.DataFrame, column_ # if the Bottle S/N column is present then use that values as the bottle ID if 'bottle_' in row[1]: - bottle_id = int(row[1]['bottle_']) + try: + bottle_id = int(row[1]['bottle_']) + except ValueError as ex: + logger.exception(ex) + message = _("There was an error parsing a bottle id - ") + message += _("\nEvent Comments : ") + for action in event.actions.all(): + if action.comment: + message += f"\n{action.get_type_display()} : {action.comment}" + + logger.error(message) + + core_models.FileError(mission=mission, file_name=file_name, line=skipped_rows, message=message, + type=core_models.ErrorType.validation).save() + continue if not bottles.filter(bottle_id=bottle_id).exists(): - message = _("Bottle does not exist for event") + message = _("Bottle does not exist for event \n") message += _("Event") + f" #{event.event_id} " + _("Bottle ID") + f" #{bottle_id}" + message += _("\nEvent Comments : ") + for action in event.actions.all(): + if action.comment: + message += f"\n{action.get_type_display()} : {action.comment}" logger.warning(message) continue diff --git a/core/tests/TestAndesParser.py b/core/tests/TestAndesParser.py index 103ea01..d6553d5 100644 --- a/core/tests/TestAndesParser.py +++ b/core/tests/TestAndesParser.py @@ -152,6 +152,7 @@ def test_andeis_parser_events(self): { "event_number": 11.0, "instrument": "Plankton net (202μm)", + "instrument_type": "NET", "wire_out": "1000 m", "wire_angle": "45 degrees", "flow_meter_start": 2068, diff --git a/core/views_mission_sample.py b/core/views_mission_sample.py index 6042488..b0da44f 100644 --- a/core/views_mission_sample.py +++ b/core/views_mission_sample.py @@ -168,7 +168,10 @@ def get_file_error_card(request, database, mission_id): li_id = f'error_{error.pk}' li = soup.new_tag('li', attrs={'class': 'list-group-item', 'id': li_id}) div = soup.new_tag('div', attrs={'class': 'col'}) - div.string = error.message + msgs = error.message.split("\n") + for msg in msgs: + div.append(msg_div:=soup.new_tag('div')) + msg_div.string = msg url = reverse_lazy('core:mission_samples_delete_file_error', args=(database, error.pk)) btn_attrs = { diff --git a/start_dart.bat b/start_dart.bat index 7eec7fe..2e57182 100644 --- a/start_dart.bat +++ b/start_dart.bat @@ -4,7 +4,7 @@ if not exist ".\logs\" ( mkdir logs ) -set dart_version=3.3.0.2 +set dart_version=3.3.1 REM if this is not a git repo, and the application was installed from zip file we just want to run update REM if this is a cloned version of the git repo we want to pull from master, then run the update diff --git a/update.bat b/update.bat index c1c4956..081f0c0 100644 --- a/update.bat +++ b/update.bat @@ -7,7 +7,7 @@ if not exist ".\logs\" ( REM If this was run from a clone repo we can force an update of the python libraries, collectstatic or a REM migration on the database by changing the update version -set update_version=3.3.0.2 +set update_version=3.3.1 set first_run=0 set server_path=.\dart_env\Scripts\activate.bat