From aa82555e433f761afead1c40672e87e002e293f8 Mon Sep 17 00:00:00 2001 From: Jody Klymak Date: Sun, 2 Oct 2022 08:47:56 -0700 Subject: [PATCH 1/2] FIX: refactor dbdreader code to just do one pass --- pyglider/slocum.py | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/pyglider/slocum.py b/pyglider/slocum.py index 323699e..233db36 100644 --- a/pyglider/slocum.py +++ b/pyglider/slocum.py @@ -789,7 +789,7 @@ def raw_to_timeseries(indir, outdir, deploymentyaml, *, def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, - search='*.[D|E]BD', + search='*.[D|E]BD', time_base='sci_water_temp', profile_filt_time=100, profile_min_time=300): """ Convert directly from binary files to netcdf timeseries file. Requires @@ -847,13 +847,28 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, for atts in ncvar[name].keys(): if atts != 'coordinates': attr[atts] = ncvar[name][atts] - if 'sci_m_present_time' in dbd.parameterNames['sci']: - time_base = 'sci_m_present_time' - elif 'm_present_time' in dbd.parameterNames['eng']: - time_base = 'm_present_time' - ds[name] = (('time'), dbd.get(time_base)[0], attr) + sensors = [time_base] - for name in thenames: + for nn, name in enumerate(thenames): + sensorname = ncvar[name]['source'] + if not sensorname == time_base: + sensors.append(sensorname) + else: + baseind = nn + + # get the data, with `time_base` as the time source that + # all other variables are synced to: + data = dbd.get_sync(*sensors) + # get the time: + time = data.pop(0) + ds['time'] = (('time'), time, attr) + # get the time_base data: + basedata = data.pop(0) + # slot the time_base variable into the right place in the + # data list: + data.insert(baseind, basedata) + + for nn, name in enumerate(thenames): _log.info('working on %s', name) if 'method' in ncvar[name].keys(): continue @@ -867,23 +882,16 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, _log.info('names: %s %s', name, sensorname) if sensorname in dbd.parameterNames['sci']: _log.debug('Sci sensorname %s', sensorname) - time, time, val = dbd.get_sync(time_base, sensorname) + val = data[nn] val = utils._zero_screen(val) val = convert(val) elif sensorname in dbd.parameterNames['eng']: _log.debug('Eng sensorname %s', sensorname) - time, time, val = dbd.get_sync(time_base, sensorname) + val = data[nn] val = convert(val) ncvar['method'] = 'linear fill' else: ValueError(f'{sensorname} not in science or eng parameter names') - if len(time) < ds.sizes['time']: - _log.info(f'{sensorname} does not have as many entries ' - 'as other variables.') - # sometimes one of the sensors has more or less data: - dsfix = xr.DataArray(val, dims='time', - coords={'time': time}) - val = dsfix.reindex_like(ds).values # make the attributes: ncvar[name].pop('coordinates', None) @@ -901,7 +909,8 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *, ds = ds.assign_coords(longitude=ds.longitude) ds = ds.assign_coords(latitude=ds.latitude) ds = ds.assign_coords(depth=ds.depth) - + # screen out-of-range times; these won't convert: + ds['time'] = ds.time.where((ds.time>0) & (ds.time<6.4e9), np.NaN) ds['time'] = (ds.time.values.astype('timedelta64[s]') + np.datetime64('1970-01-01T00:00:00')) ds = utils.fill_metadata(ds, deployment['metadata'], device_data) From c9175c0b783f6095cdd2d9c5646aaefa04b3a892 Mon Sep 17 00:00:00 2001 From: Jody Klymak Date: Sun, 2 Oct 2022 09:46:01 -0700 Subject: [PATCH 2/2] FIX: make profile work even if filt len 0 --- pyglider/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyglider/utils.py b/pyglider/utils.py index f1aba0a..e339157 100644 --- a/pyglider/utils.py +++ b/pyglider/utils.py @@ -160,10 +160,11 @@ def get_profiles_new(ds, min_dp=10.0, filt_time=100, profile_min_time=300): min_nsamples = int(profile_min_time / dt) _log.info('Filt Len %d, dt %f, min_n %d', filt_length, dt, min_nsamples) - - p = np.convolve(ds.pressure.values[good], - np.ones(filt_length) / filt_length, 'same') - _log.info('ffilt, {filt_length}') + if filt_length > 1: + p = np.convolve(ds.pressure.values[good], + np.ones(filt_length) / filt_length, 'same') + else: + p = ds.pressure.values[good] decim = int(filt_length / 3) if decim < 2: decim = 2