add scripts and comments for yml reformat

reformat_yaml/correct_yaml.py

@@ -0,0 +1,71 @@
+import datetime
+import yaml
+import pandas as pd
+from pathlib import Path
+df_cal = pd.read_csv('new_cal.csv')
+df_comments = pd.read_csv('piloting_comments.csv')
+df_missions = pd.read_csv('https://erddap.observations.voiceoftheocean.org/erddap/tabledap/meta_users_table.csv')
+expected_failures = [(70, 29), (57, 58), (57, 75)]
+
+
+def correct_yaml(yaml_path):
+    with open(yaml_path) as fin:
+        deployment = yaml.safe_load(fin)
+    meta = deployment['metadata']
+    df_glider = df_missions[df_missions['glider_serial'] == f"SEA{meta['glider_serial'].zfill(3)}"]
+    mission = df_glider[df_glider['deployment_id'] == int(meta['deployment_id'])]
+    if mission.empty:
+        if (int(meta['glider_serial']), int(meta['deployment_id'])) in expected_failures:
+            return
+        print(f"fail SEA{ meta['glider_serial']} M{meta['deployment_id']} ")
+        return
+    start = str(mission['deployment_start'].values[0])[:10]
+    devices = deployment['glider_devices']
+    for device_name, cal_dict in devices.items():
+        if cal_dict['model'] not in df_cal.model.values:
+            continue
+        df_model = df_cal[df_cal['model'] == cal_dict['model']]
+        try:
+            serial = str(int(cal_dict['serial']))
+        except ValueError:
+            serial = cal_dict['serial']
+        # fix for incorrect serial number
+        if serial == '2110556':
+            serial = '210556'
+            cal_dict['serial'] = serial
+        df_serial = df_model[df_model['serial'] == serial]
+        df_serial = df_serial.sort_values('calibration_date')
+        if cal_dict['calibration_date'] in df_serial['calibration_date'].values:
+            continue
+        df_pre_deployment_cals = df_serial[df_serial['calibration_date'] < start]
+        new_cal_date = df_pre_deployment_cals['calibration_date'].values[-1]
+        old_cal_date = cal_dict['calibration_date']
+        time_diff = (datetime.datetime.strptime(new_cal_date, '%Y-%m-%d') - datetime.datetime.strptime(old_cal_date, '%Y-%m-%d')).days
+        # fix the calibration date
+        deployment['glider_devices'][device_name]['calibration_date'] = new_cal_date
+        # Don't warn if time difference is small
+        if abs(time_diff) < 30:
+            continue
+        # Don't warn if calibration month-days are just switched american style
+        if old_cal_date == new_cal_date[:5] + new_cal_date[8:] + new_cal_date[4:7]:
+            continue
+        print(f"CORRECTION {old_cal_date} >> {new_cal_date}. {yaml_path.name.split('.')[0]} {cal_dict['model']} {serial}. {time_diff} days ")
+    df_glider_comment = df_comments[df_comments['Glider'] == f"SEA{meta['glider_serial'].zfill(3)}"]
+    df_mission_comment = df_glider_comment[df_glider_comment['Mission'] == f"M{meta['deployment_id']}"]
+    if df_mission_comment.empty:
+        print(f"no comment found for {yaml_path.name}")
+    if not df_mission_comment.empty:
+        comment = df_mission_comment['Comments'].values[0]
+        try:
+            comment = comment.replace('\n', '. ')
+        except:
+            print(f'failed to strip comment: {yaml_path} {comment}')
+        deployment['metadata']['comment'] = comment
+
+    with open(yaml_path, "w") as fout:
+        yaml.dump(deployment, fout, sort_keys=False)
+
+
+if __name__ == '__main__':
+    for yml in list(Path("../mission_yaml").glob("*.yml")):
+        correct_yaml(yml)

reformat_yaml/new_cal.csv

@@ -0,0 +1,110 @@
+model,serial,calibration_date
+legato,207476,2021-06-10
+legato,205047,2020-07-21
+legato,205047,2023-08-02
+legato,210555,2022-05-16
+legato,206523,2021-03-01
+legato,205046,2020-07-21
+legato,205046,2023-09-05
+legato,206613,2021-03-04
+legato,206613,2023-06-02
+legato,204958,2020-06-22
+legato,204958,2023-09-05
+legato,205048,2020-07-22
+legato,205048,2022-10-03
+legato,205048,2023-11-09
+legato,212534,2022-10-21
+legato,206611,2021-03-03
+legato,206612,2021-03-07
+legato,206610,2021-03-01
+GPCTD,CTD0271,2022-05-01
+legato,210556,2022-05-17
+legato,230629,2023-06-14
+legato,206524,2021-02-17
+legato,212532,2022-10-21
+legato,204069,2020-03-31
+legato,204070,2020-03-31
+legato,204072,2020-03-31
+legato,204951,2020-07-06
+FLBBCD,7564,2022-05-10
+FLBBCD,7485,2022-04-04
+FLBBCD,8201,2023-04-27
+FLBBPC,6756,2021-03-22
+FLBBPC,6754,2021-03-22
+FLBBCD,5925,2019-10-14
+FLBBPC,6598,2020-12-22
+FLNTU,6170,2020-03-02
+FLNTU,6661,2021-02-01
+FLBBCD,7619,2022-06-21
+FLBBCD,5948,2019-10-30
+FLBBPC,6597,2020-12-22
+FLBBPC,6755,2021-03-22
+FLBBPC,7313,2022-01-11
+SEAOWL,165,2022-06-01
+FLBBCD,7522,2022-04-20
+FLNTU,6134,2020-02-06
+FLBBPC,6753,2021-03-22
+FLBBCD,7563,2022-05-10
+coda,205592,2021-02-26
+coda,205592,2022-07-12
+coda,206112,2021-02-26
+coda,206112,2022-22-15
+coda,206115,2021-02-26
+coda,206116,2021-02-26
+coda,206116,2022-07-12
+coda,206116,2023-06-23
+coda,206795,2021-06-22
+coda,206795,2021-09-15
+coda,206795,2022-07-12
+coda,210568,2022-05-26
+coda,210577,2022-05-13
+coda,212280,2022-11-02
+coda,212285,2022-11-02
+AROD-FT,67,2022-11-11
+AROD-FT,67,2023-08-26
+AROD-FT,41,2020-02-08
+AROD-FT,66,2022-11-08
+AROD-FT,70,2022-11-11
+AROD-FT,70,2023-08-26
+AROD-FT,50,2022-03-19
+AROD-FT,68,2022-11-08
+AROD-FT,40,2020-02-05
+AROD-FT,40,2023-08-11
+AROD-FT,58,2022-03-26
+AROD-FT,58,2023-08-26
+AROD-FT,39,2020-02-05
+AROD-FT,44,2021-01-23
+AROD-FT,71,2022-11-26
+AROD-FT,69,2022-11-11
+AROD-FT,81,2023-04-01
+AROD-FT,38,2020-02-05
+AROD-FT,42,2020-05-23
+AROD-FT,43,2021-01-20
+AD2CP,101136,2020-01-24
+AD2CP,101589,2021-02-02
+AD2CP,101593,2020-01-24
+AD2CP,102758,2021-02-03
+AD2CP,102882,2021-05-21
+AD2CP,103354,2022-01-11
+AD2CP,103355,2022-03-17
+AD2CP,103356,2022-03-15
+AD2CP,103356,2023-10-19
+AD2CP,103360,2022-03-15
+AD2CP,103363,2022-05-25
+AD2CP,103364,2022-05-01
+AD2CP,104646,2022-07-18
+AD2CP,104648,2023-08-16
+AD2CP,104649,2022-07-18
+AD2CP,101594,2020-09-30
+AD2CP,105526,2023-09-04
+Deep SUNA,NTR-1293,2019-05-20
+Nanoflu,D54D,2021-05-20
+MPE-PAR,2984,2023-02-21
+MPE-PAR,2983,2023-02-21
+MPE-PAR,2874,2022-08-11
+OCR504,2043,2020-02-25
+OCR504,2044,2020-02-21
+OCR504,2197,2021-03-08
+OCR504,2199,2021-03-15
+OCR504,2193,2021-02-02
+METS,2117,2022-07-01