Skip to content

Commit

Permalink
Merge pull request #1360 from MBARIMike/master
Browse files Browse the repository at this point in the history 
Updates developed on kraken for stoqs_denmark2024, stoqs_rovctd_mw93, and stoqs_greatlakes2024
  • Loading branch information
@stoqs
stoqs authored Nov 26, 2024
2 parents 1084033 + 90aa471 commit daf5810
Show file tree
Hide file tree
Showing 13 changed files with 309 additions and 37 deletions.
1 change: 1 addition & 0 deletions stoqs/loaders/CANON/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@ def loadLRAUV(self, pname, startdate=None, enddate=None,
'concentration_of_colored_dissolved_organic_matter_in_sea_water',
'bin_mean_concentration_of_colored_dissolved_organic_matter_in_sea_water',
'bin_median_concentration_of_colored_dissolved_organic_matter_in_sea_water',
'current_direction_navigation_frame', 'current_speed_navigation_frame',
],
stride=None, file_patterns=('.*2S_scieng.nc$'), build_attrs=True,
dlist_str=None, err_on_missing_file=False, critSimpleDepthTime=10,
Expand Down
2 changes: 2 additions & 0 deletions stoqs/loaders/CANON/toNetCDF/lrauvNc4ToNetcdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1371,6 +1371,8 @@ def processResampleNc4File(self, in_file, out_file, parm, resampleFreq, rad_to_d
value = value.mask(value > 1000, np.nan) # Remove the 5 depth values greater than 1000 m
if 'triton/missionlogs/2023/20230909_20230918/20230918T035454/202309180355_202309181709' in in_file:
value = value.mask(value > 20, np.nan) # Remove 477.49 m value in shallow water of Lake Erie
if 'makai/missionlogs/2024/20240607_20240615/20240611T082709/202406110827_202406111026' in in_file:
value = value.mask(value > 20, np.nan) # Remove 230 m values in shallow water of North Sea in stoqs_denmark2024

i = self.interpolate(value, t_resample.index)
if key == 'time':
Expand Down
7 changes: 6 additions & 1 deletion stoqs/loaders/CANON/toNetCDF/makeLRAUVNetCDFs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,12 @@
'RudderServo': [{'name': 'platform_rudder_angle',
'rename': 'control_inputs_rudder_angle'}],
'ThrusterServo': [{'name': 'platform_propeller_rotation_rate',
'rename': 'control_inputs_propeller_rotation_rate'}]}
'rename': 'control_inputs_propeller_rotation_rate'}],
'CurrentEstimator': [{'name': 'current_direction_navigation_frame',
'rename': 'current_direction_navigation_frame'},
{'name': 'current_speed_navigation_frame',
'rename': 'current_speed_navigation_frame'}],
}

SCIENG_PARMS = {**SCI_PARMS, **ENG_PARMS}

Expand Down
105 changes: 105 additions & 0 deletions stoqs/loaders/Denmark/load_2024.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
#!/usr/bin/env python
'''
MBARI ESP Team
Loader for Lake Erie Brizo and Makai 2021 deployments
Brizo Lake Erie: 20210805_20210816.dlist
Makai Lake Erie: 20210804_20210808.dlist
Mike McCann, Duane Edgington, Danelle Cline
MBARI 1 September 2021
'''

import os
import sys

parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found

from CANON import CANONLoader
from datetime import datetime
import timing

cl = CANONLoader('stoqs_denmark2024_fix', 'Denmark LRAUV Deployments - June 2024',
description='LRAUV Deployments Deployments near Denmark in 2024',
x3dTerrains = {
'https://stoqs.mbari.org/x3d/Globe_1m_bath_10x/Globe_1m_bath_10x_scene.x3d': {
'position': '14051448.48336 -15407886.51486 6184041.22775',
'orientation': '0.83940 0.33030 0.43164 1.44880',
'centerOfRotation': '0 0 0',
'VerticalExaggeration': '10',
},
# 'https://stoqs.mbari.org/x3d/michigan_lld_10x/michigan_lld_10x_src_scene.x3d': {
# 'position': '260508.70011 -4920934.73726 4257472.30926',
# 'orientation': '1.00000 0.00113 -0.00025 1.66609',
# 'centerOfRotation': '239815.39152 -4691345.97184 4297950.38875',
# 'VerticalExaggeration': '10',
# 'speed': '0.1',
# },
# Need to convert .grd to legacy format for mbgrd2gtlf:
# gmt grdconvert michigan_lld.grd -Gmichigan_lld_new.grd=cd
# mbgrd2gltf michigan_lld_new.grd -e 100 -b
# mv michigan_lld_new.glb michigan_lld_100x.glb # Then copy to stoqs.mbari.org
# 'https://stoqs.mbari.org/x3d/michigan_lld_100x/michigan_lld_100x.glb': {
# 'position': '260508.70011 -4920934.73726 4257472.30926',
# 'orientation': '1.00000 0.00113 -0.00025 1.66609',
# 'centerOfRotation': '239815.39152 -4691345.97184 4297950.38875',
# 'VerticalExaggeration': '100',
# 'speed': '0.1',
# },
# TODO: Add glb model for Lake Superior
},
grdTerrain = os.path.join(parentDir, 'michigan_lld.grd')
)

makai_sdate = datetime(2024, 6, 5)
makai_edate = datetime(2024, 6, 19)
brizo_sdate = datetime(2024, 6, 5)
brizo_edate = datetime(2024, 6, 19)

cl.process_command_line()

if cl.args.test:
cl.stride = 1
# Uncomment the test to use...

# Test loading brizo's ESP Cartridges 29 to 23
# cl.brizo_base = 'http://dods.mbari.org/opendap/data/lrauv/brizo/missionlogs/2024/20240607_20240615/20240612T184249/'
# cl.brizo_files = ['202406121843_202406130908_2S_scieng.nc']
# cl.brizo_parms =['temperature', 'current_direction_navigation_frame', 'current_speed_navigation_frame']
# cl.loadLRAUV('brizo', brizo_sdate, brizo_edate, critSimpleDepthTime=0.1, build_attrs=False)

# Test loading makai's ESP Cartridges 59/58
# https://dods.mbari.org/opendap/data/lrauv/makai/missionlogs/2024/20240607_20240615/20240607T125711/202406071257_202406080957_2S_scieng.nc.html
# cl.makai_base = 'http://dods.mbari.org/opendap/data/lrauv/makai/missionlogs/2024/20240607_20240615/20240607T125711/'
# cl.makai_files = ['202406071257_202406080957_2S_scieng.nc']
# cl.makai_parms =['temperature', 'current_direction_navigation_frame', 'current_speed_navigation_frame']
# cl.loadLRAUV('makai', makai_sdate, makai_edate, critSimpleDepthTime=0.1, build_attrs=False)

# Test loading makai's ESP Cartridges 56/55
cl.makai_base = 'http://dods.mbari.org/opendap/data/lrauv/makai/missionlogs/2024/20240607_20240615/20240608T095744/'
cl.makai_files = ['202406080957_202406082214_2S_scieng.nc ']
cl.makai_parms =['temperature', 'current_direction_navigation_frame', 'current_speed_navigation_frame']
cl.loadLRAUV('makai', makai_sdate, makai_edate, critSimpleDepthTime=0.1, build_attrs=False)

# Test Duplicate data and missing/overlapping Cartridges for makai
# makai_sdate = datetime(2024, 6, 7)
# makai_edate = datetime(2024, 6, 16)
# cl.loadLRAUV('makai', makai_sdate, makai_edate, critSimpleDepthTime=0.1, build_attrs=True)

# cl.loadLRAUV('makai', makai_sdate, makai_edate, critSimpleDepthTime=0.1)
# cl.loadLRAUV('brizo', brizo_sdate, brizo_edate, critSimpleDepthTime=0.1)
elif cl.args.stride:
cl.stride = cl.args.stride
# Realtime sbd logs load
# cl.loadLRAUV('makai', makai_sdate, makai_edate, critSimpleDepthTime=0.1, sbd_logs=True)
# cl.loadLRAUV('brizo', brizo_sdate, brizo_edate, critSimpleDepthTime=0.1, sbd_logs=True)
# Post recovery missionlogs load
cl.loadLRAUV('makai', makai_sdate, makai_edate, critSimpleDepthTime=0.1)
cl.loadLRAUV('brizo', brizo_sdate, brizo_edate, critSimpleDepthTime=0.1)

# Add any X3D simulation anbd Terrain information specified in the constructor to the database - must be done after a load is executed
cl.addTerrainResources()

print("All Done.")

62 changes: 62 additions & 0 deletions stoqs/loaders/GreatLakes/load_2024.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env python

import os
import sys

parentDir = os.path.join(os.path.dirname(__file__), "../")
sys.path.insert(0, parentDir) # So that CANON is found

from CANON import CANONLoader
from datetime import datetime
import timing

cl = CANONLoader('stoqs_greatlakes2024', 'Great Lakes LRAUV Deployments - July-August 2024',
description='Great Lakes LRAUV Deployments Deployments in 2024',
x3dTerrains = {
# 'https://stoqs.mbari.org/x3d/Globe_1m_bath_10x/Globe_1m_bath_10x_scene.x3d': {
# 'position': '14051448.48336 -15407886.51486 6184041.22775',
# 'orientation': '0.83940 0.33030 0.43164 1.44880',
# 'centerOfRotation': '0 0 0',
# 'VerticalExaggeration': '10',
# },
# 'https://stoqs.mbari.org/x3d/michigan_lld_10x/michigan_lld_10x_src_scene.x3d': {
# 'position': '260508.70011 -4920934.73726 4257472.30926',
# 'orientation': '1.00000 0.00113 -0.00025 1.66609',
# 'centerOfRotation': '239815.39152 -4691345.97184 4297950.38875',
# 'VerticalExaggeration': '10',
# 'speed': '0.1',
# },
# Need to convert .grd to legacy format for mbgrd2gtlf:
# gmt grdconvert michigan_lld.grd -Gmichigan_lld_new.grd=cd
# mbgrd2gltf michigan_lld_new.grd -e 100 -b
# mv michigan_lld_new.glb michigan_lld_100x.glb # Then copy to stoqs.mbari.org
'https://stoqs.mbari.org/x3d/michigan_lld_100x/michigan_lld_100x.glb': {
'position': '260508.70011 -4920934.73726 4257472.30926',
'orientation': '1.00000 0.00113 -0.00025 1.66609',
'centerOfRotation': '239815.39152 -4691345.97184 4297950.38875',
'VerticalExaggeration': '100',
'speed': '0.1',
},
# TODO: Add glb model for Lake Superior
},
grdTerrain = os.path.join(parentDir, 'michigan_lld.grd')
)

triton_sdate = datetime(2024, 7, 20)
triton_edate = datetime(2024, 8, 12)

cl.process_command_line()

if cl.args.test:
cl.stride = 100
cl.loadLRAUV('triton', triton_sdate, triton_edate, critSimpleDepthTime=0.1, build_attrs=True)
elif cl.args.stride:
cl.stride = cl.args.stride
# Post recovery missionlogs load
cl.loadLRAUV('triton', triton_sdate, triton_edate, critSimpleDepthTime=0.1)

# Add any X3D simulation anbd Terrain information specified in the constructor to the database - must be done after a load is executed
cl.addTerrainResources()

print("All Done.")

3 changes: 2 additions & 1 deletion stoqs/loaders/PlanktonProxies/load_all_dorado.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@
startdate = datetime.strptime(cl.args.startdate, '%Y%m%d')
if cl.args.enddate:
enddate = datetime.strptime(cl.args.enddate, '%Y%m%d')
cl.loadDorado(startdate, enddate, build_attrs=True, plankton_proxies=True)
cl.loadDorado(startdate, enddate, build_attrs=True, file_patterns=(r".*netcdf/dorado_.*1S.nc", ))
#cl.loadDorado(startdate, enddate, build_attrs=True, file_patterns=(r".*_decim.nc$", ), plankton_proxies=True)

# Add any X3D Terrain information specified in the constructor to the database - must be done after a load is executed
cl.addTerrainResources()
Expand Down
2 changes: 1 addition & 1 deletion stoqs/loaders/ROVCTD/ROVCTDloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ def _pymssqlStartAndEndTimes(self):
logger.debug(f"{r = }")
sdt = datetime.strptime(r['divestartdtg'].strip(), '%Y-%m-%dT%H:%M:%S')
edt = datetime.strptime(r['diveenddtg'].strip(), '%Y-%m-%dT%H:%M:%S')
except TypeError:
except (TypeError, AttributeError):
raise DiveInfoServletException('Cannot get start and end times for %s%d' % (self.platformName[0].upper(), self.diveNumber))

return sdt, edt
Expand Down
12 changes: 9 additions & 3 deletions stoqs/loaders/ROVCTD/loadAllTransectDives.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
# V541 T511 D238
# ... ... ...

./ROVCTDloader.py --database stoqs_rovctd_mw93 --dives \
# Added more dives from email sent from Rob on 27 August 2024

stoqs/loaders/ROVCTD/ROVCTDloader.py --database stoqs_rovctd_mw93 --dives \
V526 V535 V537 V541 V581 V582 V585 V586 V591 V595 V600 V619 V636 V637 V642 \
V656 V659 V670 V682 V685 V690 V695 V702 V709 V710 V717 V745 V753 V754 V757 \
V762 V766 V767 V778 V783 V789 V798 V805 V818 V821 V822 V829 V849 V856 V859 \
Expand All @@ -49,10 +51,14 @@ V3006 V3018 V3066 V3079 V3092 V3130 V3142 V3157 V3168 V3181 V3196 V3205 V3229 \
V3249 V3267 V3285 V3291 V3313 V3320 V3334 V3354 V3363 V3385 V3409 V3417 V3429 \
V3453 V3463 V3481 V3514 V3536 V3559 V3571 V3595 V3600 V3603 V3607 V3621 V3630 \
V3646 V3656 V3668 V3682 V3685 V3692 V3700 V3707 V3715 V3725 V3731 V3736 V3742 \
V3715 V3725 V3731 V3736 V3742 \
V3748 V3750 V3755 V3766 V3767 V3774 V3781 V3804 V3821 \
V3822 V3827 V3836 V3848 V3862 V3880 V3886 V3900 V3908 V3931 V3937 V3948 V3954 \
V3974 V3986 V3992 V3995 V4006 V4029 V4044 V4047 V4068 V4080 V4329 \
T103 T257 T411 T511 T624 T648 T649 T650 T683 T686 T687 T688 T762 T763 T764 \
T835 T836 T837 T894 T895 T896 T995 T1111 \
D147 D181 D200 D238 D311 D416 D433 D449 D478 D544 D595 D646 D691 \
--campaignName 'Midwater 1 transect dives - 2014' \
--campaignDescription 'Daytime transect dives made by the Robison lab at Midwater 1 site, 1993 - 2014'
D787 D832 D870 D907 D924 D996 \
--campaignName 'Midwater 1 transect dives - 2024' \
--campaignDescription 'Daytime transect dives made by the Robison lab at Midwater 1 site, 1993 - 2024'

Loading

0 comments on commit daf5810

Please sign in to comment.