diff --git a/pipelines/toast_bin_so3g.py b/pipelines/toast_bin_so3g.py index 280f862cf..dd7c14952 100644 --- a/pipelines/toast_bin_so3g.py +++ b/pipelines/toast_bin_so3g.py @@ -16,7 +16,7 @@ import toast.tod import toast.pipeline_tools as toast_tools -from sotodlib.data.toast_load import load_data +from sotodlib.io.toast_load import load_data def main(): diff --git a/pipelines/toast_so_example.py b/pipelines/toast_so_example.py index 0bad4d0e6..7eb860690 100755 --- a/pipelines/toast_so_example.py +++ b/pipelines/toast_so_example.py @@ -13,8 +13,8 @@ import toast.map as tm import toast.tod as tt -from sotodlib.hardware import get_example, sim_telescope_detectors -from sotodlib.data.toast_load import load_data +from sotodlib.sim_hardware import get_example, sim_telescope_detectors +from sotodlib.io.toast_load import load_data def binned_map(data, npix, subnpix, out="."): diff --git a/pipelines/toast_so_sim.py b/pipelines/toast_so_sim.py index db9370d3c..ba2f41c80 100755 --- a/pipelines/toast_so_sim.py +++ b/pipelines/toast_so_sim.py @@ -40,12 +40,10 @@ import toast.pipeline_tools as toast_tools -import sotodlib.pipeline_tools as so_tools +import sotodlib.utils.pipeline_tools as so_tools import numpy as np -import sotodlib.hardware - import warnings warnings.filterwarnings("ignore") diff --git a/pipelines/toast_so_tf.py b/pipelines/toast_so_tf.py index a3bfa9665..f34f59bb1 100644 --- a/pipelines/toast_so_tf.py +++ b/pipelines/toast_so_tf.py @@ -31,12 +31,10 @@ import toast.pipeline_tools as toast_tools -import sotodlib.pipeline_tools as so_tools +import sotodlib.utils.pipeline_tools as so_tools import numpy as np -import sotodlib.hardware - import warnings warnings.filterwarnings("ignore") diff --git a/sotodlib/__init__.py b/sotodlib/__init__.py index 4d4763eb8..62fddc38f 100644 --- a/sotodlib/__init__.py +++ b/sotodlib/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2019 Simons Observatory. +# Copyright (c) 2018-2020 Simons Observatory. # Full license can be found in the top level "LICENSE" file. """Simons Observatory TOD Simulation and Processing. @@ -8,9 +8,11 @@ Contents: - * db: database access for detector properties, observations, etc. - * toast: tools specifically for working with the external TOAST package. - * scripts: commandline entry points. + * core: Container classes for data, metadata, etc. + * io: Code for reading / writing data. + * utils: General utility functions and data helper functions. + * scripts: Commandline entry points. + * Top level packages for actual data processing. """ diff --git a/sotodlib/core/__init__.py b/sotodlib/core/__init__.py index 13d17abe1..f80f66b27 100644 --- a/sotodlib/core/__init__.py +++ b/sotodlib/core/__init__.py @@ -1,4 +1,13 @@ +# Copyright (c) 2018-2020 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +"""Simons Observatory core routines. + +This module has containers and in-memory structures for data and metadata. + +""" +from .context import Context + from .axisman import AxisManager from .axisman import IndexAxis, OffsetAxis, LabelAxis -from .context import Context +from .hardware import Hardware diff --git a/sotodlib/core/context.py b/sotodlib/core/context.py index d89fbb637..ef1ca2a7c 100644 --- a/sotodlib/core/context.py +++ b/sotodlib/core/context.py @@ -2,7 +2,7 @@ import yaml import os -from .. import metadata +from . import metadata class Context(odict): @@ -77,11 +77,11 @@ def reload(self, load_list='all'): """ # Metadata support databases. - for key, cls in [('detdb', metadata.DetDB), - ('obsdb', metadata.ObsDB), - ('obsfiledb', metadata.ObsFileDB)]: + for key, cls in [('detdb', metadata.DetDb), + ('obsdb', metadata.ObsDb), + ('obsfiledb', metadata.ObsFileDb)]: if (load_list == 'all' or key in load_list) and key in self: - # E.g. self.detdb = DetDB.from_file(self['detdb'] + # E.g. self.detdb = DetDb.from_file(self['detdb'] db = cls.from_file(self[key]) setattr(self, key, db) # The metadata loader. diff --git a/sotodlib/data/core.py b/sotodlib/core/g3_core.py similarity index 100% rename from sotodlib/data/core.py rename to sotodlib/core/g3_core.py diff --git a/sotodlib/hardware/config.py b/sotodlib/core/hardware.py similarity index 50% rename from sotodlib/hardware/config.py rename to sotodlib/core/hardware.py index 495ff0dd6..80299ad0c 100644 --- a/sotodlib/hardware/config.py +++ b/sotodlib/core/hardware.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2019 Simons Observatory. +# Copyright (c) 2018-2020 Simons Observatory. # Full license can be found in the top level "LICENSE" file. """Hardware configuration utilities. """ @@ -265,354 +265,3 @@ def select(self, telescopes=None, tubes=None, match=dict()): hw.data["detectors"] = newdets return hw - - -def get_example(): - """Return an example Hardware config with the required sections. - - The returned Hardware object has 4 fake detectors as an example. These - detectors can be replaced by the results of other simulation functions. - - Returns: - (Hardware): Hardware object with example parameters. - - """ - cnf = OrderedDict() - - bands = OrderedDict() - - bnd = OrderedDict() - bnd["center"] = 25.7 - bnd["low"] = 21.7 - bnd["high"] = 29.7 - bnd["bandpass"] = "" - bnd["NET"] = 300.0 - bnd["fknee"] = 50.0 - bnd["fmin"] = 0.01 - bnd["alpha"] = 3.5 - # Noise elevation scaling fits from Carlos Sierra - # These numbers are for V3 LAT baseline - bnd["A"] = 0.09 - bnd["C"] = 0.87 - bands["LF1"] = bnd - - bnd = OrderedDict() - bnd["center"] = 38.9 - bnd["low"] = 30.9 - bnd["high"] = 46.9 - bnd["bandpass"] = "" - bnd["NET"] = 300.0 - bnd["fknee"] = 50.0 - bnd["fmin"] = 0.01 - bnd["alpha"] = 3.5 - bnd["A"] = 0.25 - bnd["C"] = 0.64 - bands["LF2"] = bnd - - bnd = OrderedDict() - bnd["center"] = 92.0 - bnd["low"] = 79.0 - bnd["high"] = 105.0 - bnd["bandpass"] = "" - bnd["NET"] = 300.0 - bnd["fknee"] = 50.0 - bnd["fmin"] = 0.01 - bnd["alpha"] = 3.5 - bnd["A"] = 0.14 - bnd["C"] = 0.80 - bands["MFF1"] = bnd - - bnd = OrderedDict() - bnd["center"] = 147.5 - bnd["low"] = 130.0 - bnd["high"] = 165.0 - bnd["bandpass"] = "" - bnd["NET"] = 400.0 - bnd["fknee"] = 50.0 - bnd["fmin"] = 0.01 - bnd["alpha"] = 3.5 - bnd["A"] = 0.17 - bnd["C"] = 0.76 - bands["MFF2"] = bnd - - bnd = OrderedDict() - bnd["center"] = 88.6 - bnd["low"] = 75.6 - bnd["high"] = 101.6 - bnd["bandpass"] = "" - bnd["NET"] = 300.0 - bnd["fknee"] = 50.0 - bnd["fmin"] = 0.01 - bnd["alpha"] = 3.5 - bnd["A"] = 0.19 - bnd["C"] = 0.74 - bands["MFS1"] = bnd - - bnd = OrderedDict() - bnd["center"] = 146.5 - bnd["low"] = 128.0 - bnd["high"] = 165.0 - bnd["bandpass"] = "" - bnd["NET"] = 400.0 - bnd["fknee"] = 50.0 - bnd["fmin"] = 0.01 - bnd["alpha"] = 3.5 - bnd["A"] = 0.19 - bnd["C"] = 0.73 - bands["MFS2"] = bnd - - bnd = OrderedDict() - bnd["center"] = 225.7 - bnd["low"] = 196.7 - bnd["high"] = 254.7 - bnd["bandpass"] = "" - bnd["NET"] = 400.0 - bnd["fknee"] = 50.0 - bnd["fmin"] = 0.01 - bnd["alpha"] = 3.5 - bnd["A"] = 0.30 - bnd["C"] = 0.58 - bands["UHF1"] = bnd - - bnd = OrderedDict() - bnd["center"] = 285.4 - bnd["low"] = 258.4 - bnd["high"] = 312.4 - bnd["bandpass"] = "" - bnd["NET"] = 400.0 - bnd["fknee"] = 50.0 - bnd["fmin"] = 0.01 - bnd["alpha"] = 3.5 - bnd["A"] = 0.36 - bnd["C"] = 0.49 - bands["UHF2"] = bnd - - cnf["bands"] = bands - - wafers = OrderedDict() - - wtypes = ["UHF", "MFF", "MFS", "LF"] - wcnt = { - "LF": 1*7 + 1*3, - "MFF": 1*7 + 2*3, - "MFS": 1*7 + 2*3, - "UHF": 1*7 + 2*3 - } - wnp = { - "LF": 37, - "MFF": 432, - "MFS": 397, - "UHF": 432 - } - wpixmm = { - "LF": 18.0, - "MFF": 5.3, - "MFS": 5.6, - "UHF": 5.3 - } - wrhombgap = { - "MFF": 0.71, - "UHF": 0.71, - } - wbd = { - "LF": ["LF1", "LF2"], - "MFF": ["MFF1", "MFF2"], - "MFS": ["MFS1", "MFS2"], - "UHF": ["UHF1", "UHF2"] - } - windx = 0 - cardindx = 0 - for wt in wtypes: - for ct in range(wcnt[wt]): - wn = "{:02d}".format(windx) - wf = OrderedDict() - wf["type"] = wt - if (wt == "LF") or (wt == "MFS"): - wf["packing"] = "S" - else: - wf["packing"] = "F" - wf["rhombusgap"] = wrhombgap[wt] - wf["npixel"] = wnp[wt] - wf["pixsize"] = wpixmm[wt] - wf["bands"] = wbd[wt] - wf["card"] = "{:02d}".format(cardindx) - cardindx += 1 - wafers[wn] = wf - windx += 1 - - cnf["wafers"] = wafers - - tubes = OrderedDict() - - woff = { - "LF": 0, - "MFF": 0, - "MFS": 0, - "UHF": 0 - } - - ltubes = ["UHF", "UHF", "MFF", "MFF", "MFS", "MFS", "LF"] - ltubepos = [0, 1, 2, 3, 5, 6, 10] - for tindx in range(7): - nm = "LT{:d}".format(tindx) - ttyp = ltubes[tindx] - tb = OrderedDict() - tb["type"] = ttyp - tb["waferspace"] = 127.89 - tb["wafers"] = list() - for tw in range(3): - off = 0 - for w, props in cnf["wafers"].items(): - if props["type"] == ttyp: - if off == woff[ttyp]: - tb["wafers"].append(w) - woff[ttyp] += 1 - break - off += 1 - tb["location"] = ltubepos[tindx] - tubes[nm] = tb - - stubes = ["UHF", "MFF", "MFS", "LF"] - for tindx in range(4): - nm = "ST{:d}".format(tindx) - ttyp = stubes[tindx] - tb = OrderedDict() - tb["type"] = ttyp - tb["waferspace"] = 127.89 - tb["wafers"] = list() - for tw in range(7): - off = 0 - for w, props in cnf["wafers"].items(): - if props["type"] == ttyp: - if off == woff[ttyp]: - tb["wafers"].append(w) - woff[ttyp] += 1 - break - off += 1 - tb["location"] = 0 - tubes[nm] = tb - - cnf["tubes"] = tubes - - telescopes = OrderedDict() - - tele = OrderedDict() - tele["tubes"] = ["LT0", "LT1", "LT2", "LT3", "LT4", "LT5", "LT6"] - tele["platescale"] = 0.00495 - # This tube spacing in mm corresponds to 1.78 degrees projected on - # the sky at a plate scale of 0.00495 deg/mm. - tele["tubespace"] = 359.6 - fwhm = OrderedDict() - fwhm["LF1"] = 7.4 - fwhm["LF2"] = 5.1 - fwhm["MFF1"] = 2.2 - fwhm["MFF2"] = 1.4 - fwhm["MFS1"] = 2.2 - fwhm["MFS2"] = 1.4 - fwhm["UHF1"] = 1.0 - fwhm["UHF2"] = 0.9 - tele["fwhm"] = fwhm - telescopes["LAT"] = tele - - sfwhm = OrderedDict() - scale = 0.09668 / 0.00495 - for k, v in fwhm.items(): - sfwhm[k] = float(int(scale * v * 10.0) // 10) - - tele = OrderedDict() - tele["tubes"] = ["ST0"] - tele["platescale"] = 0.09668 - tele["fwhm"] = sfwhm - telescopes["SAT0"] = tele - - tele = OrderedDict() - tele["tubes"] = ["ST1"] - tele["platescale"] = 0.09668 - tele["fwhm"] = sfwhm - telescopes["SAT1"] = tele - - tele = OrderedDict() - tele["tubes"] = ["ST2"] - tele["platescale"] = 0.09668 - tele["fwhm"] = sfwhm - telescopes["SAT2"] = tele - - tele = OrderedDict() - tele["tubes"] = ["ST3"] - tele["platescale"] = 0.09668 - tele["fwhm"] = sfwhm - telescopes["SAT3"] = tele - - cnf["telescopes"] = telescopes - - cards = OrderedDict() - crates = OrderedDict() - - crt_indx = 0 - - for tel in cnf["telescopes"]: - crn = "{:d}".format(crt_indx) - crt = OrderedDict() - crt["cards"] = list() - crt["telescope"] = tel - - ## get all the wafer card numbers for a telescope - tb_wfrs = [cnf["tubes"][t]["wafers"] for t in cnf["telescopes"][tel]["tubes"]] - tl_wfrs = [i for sl in tb_wfrs for i in sl] - wafer_cards = [cnf["wafers"][w]["card"] for w in tl_wfrs] - - # add all cards to the card table and assign to crates - for crd in wafer_cards: - cdprops = OrderedDict() - cdprops["nbias"] = 12 - cdprops["ncoax"] = 2 - cdprops["nchannel"] = 2000 - cards[crd] = cdprops - - crt["cards"].append(crd) - - # name new crates when current one is full - if ('S' in tel and len(crt["cards"]) >=4) or len(crt["cards"]) >=6: - crates[crn] = crt - crt_indx += 1 - crn = "{:d}".format(crt_indx) - crt = OrderedDict() - crt["cards"] = list() - crt["telescope"] = tel - - # each telescope starts with a new crate - crates[crn] = crt - crt_indx += 1 - - cnf["cards"] = cards - cnf["crates"] = crates - - pl = ["A", "B"] - hand = ["L", "R"] - - dets = OrderedDict() - for d in range(4): - dprops = OrderedDict() - dprops["wafer"] = "42" - dprops["ID"] = d - dprops["pixel"] = "000" - bindx = d % 2 - dprops["band"] = "LF{}".format(bindx) - dprops["fwhm"] = 1.0 - dprops["pol"] = pl[bindx] - dprops["handed"] = hand[bindx] - dprops["card"] = "42" - dprops["channel"] = d - dprops["coax"] = 0 - dprops["bias"] = 0 - dprops["quat"] = np.array([0.0, 0.0, 0.0, 1.0]) - dname = "{}_{}_{}_{}".format("42", "000", dprops["band"], - dprops["pol"]) - dets[dname] = dprops - - cnf["detectors"] = dets - - hw = Hardware() - hw.data = cnf - - return hw diff --git a/sotodlib/core/metadata.py b/sotodlib/core/metadata.py new file mode 100644 index 000000000..f6a3d31a3 --- /dev/null +++ b/sotodlib/core/metadata.py @@ -0,0 +1,10 @@ +# Copyright (c) 2018-2020 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +"""Metadata containers. +""" + +# Import classes from sotoddb, with a rename DB -> Db. These will +# live in sotodlib soon. +from sotoddb import ResultSet, DetDB as DetDb, ObsDB as ObsDb, ObsFileDB as ObsFileDb +from sotoddb import ManifestDB as ManifestDb, ManifestScheme +from sotoddb import SuperLoader diff --git a/sotodlib/data/__init__.py b/sotodlib/data/__init__.py deleted file mode 100644 index 81aeaf04f..000000000 --- a/sotodlib/data/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2018-2019 Simons Observatory. -# Full license can be found in the top level "LICENSE" file. -"""Simons Observatory Data Processing. - -This module contains data I/O and processing tools. - -""" - -from .core import DataG3Module - -from .filter import Filter \ No newline at end of file diff --git a/sotodlib/data/condition.py b/sotodlib/g3_condition.py similarity index 88% rename from sotodlib/data/condition.py rename to sotodlib/g3_condition.py index a7f3ed2e0..8f75d4070 100644 --- a/sotodlib/data/condition.py +++ b/sotodlib/g3_condition.py @@ -12,7 +12,7 @@ import scipy.signal as signal from spt3g import core -from sotodlib.data import DataG3Module +from sotodlib.g3_core import DataG3Module class MeanSubtract(DataG3Module): def process(self, data, det_name): @@ -28,8 +28,8 @@ class Detrend(DataG3Module): resulting data can be easily re-trended (if, for example the detrend is done just for filtering). """ - - def __init__(self, input='signal', output=None, + + def __init__(self, input='signal', output=None, info='detrend_values', type='linear'): """ Args: @@ -45,27 +45,27 @@ def __init__(self, input='signal', output=None, raise ValueError("type must be 'linear' or 'constant'") self.info = info super().__init__(input, output) - + def __call__(self, f): if f.type == core.G3FrameType.Scan: self.detrend_vals = core.G3MapVectorDouble() super().__call__(f) - + if f.type == core.G3FrameType.Scan: f[self.info] = self.detrend_vals - + def process(self, data, det_name): x=np.arange(data.n_samples) self.detrend_vals[det_name] = np.polyfit(x, data, deg=self.deg) return data - np.polyval(self.detrend_vals[det_name], x) - + class Retrend(DataG3Module): """ Module for Retrending data that was Detrended with Detrend """ - def __init__(self, input='signal', output=None, + def __init__(self, input='signal', output=None, detrend_info='detrend_values'): """ Args: @@ -74,57 +74,57 @@ def __init__(self, input='signal', output=None, self.info = detrend_info super().__init__(input, output) - + def __call__(self, f): if f.type == core.G3FrameType.Scan: if self.info not in f.keys(): raise ValueError('No Detrending information in {}'.format(self.info)) - else: + else: self.retrend = f[self.info] - + super().__call__(f) - + if f.type == core.G3FrameType.Scan: f.pop(self.info) - + def process(self, data, det_name): x=np.arange(data.n_samples) return data + np.polyval(self.retrend[det_name], x) - + class Decimate(DataG3Module): """ - Module for decimating data. Uses scipy.signal.decimate() + Module for decimating data. Uses scipy.signal.decimate() """ def __init__(self, input='signal', output=None, q=5, **kwargs): """ Arguments: q (int): The downsampling factor - kwargs: can include any of the optional parameters for + kwargs: can include any of the optional parameters for scipy.signal.decimate """ self.decimate_params = {'q':q} self.decimate_params.update(kwargs) - + super().__init__(input, output) - + def process(self, data, det_name): return signal.decimate(data, **self.decimate_params) - + class Resample(DataG3Module): """ - Module for resampling data. Uses scipy.signal.resample() + Module for resampling data. Uses scipy.signal.resample() """ def __init__(self, input='signal', output=None, num=3000, **kwargs): """ Arguments: num (int): The number of samples in the resampled signal. - kwargs: can include any of the optional parameters for + kwargs: can include any of the optional parameters for scipy.signal.resample """ self.resample_params = {'num':num} self.resample_params.update(kwargs) - + super().__init__(input, output) - + def process(self, data, det_name): - return signal.resample(data, **self.resample_params) \ No newline at end of file + return signal.resample(data, **self.resample_params) diff --git a/sotodlib/data/filter.py b/sotodlib/g3_filter.py similarity index 94% rename from sotodlib/data/filter.py rename to sotodlib/g3_filter.py index af058ad75..4695c17e9 100644 --- a/sotodlib/data/filter.py +++ b/sotodlib/g3_filter.py @@ -9,30 +9,30 @@ import scipy.signal as signal from spt3g import core -from sotodlib.data import DataG3Module +from sotodlib.g3_core import DataG3Module class Filter(DataG3Module): """ G3Module that takes the G3Timestream map and applies generic filter - + Attributes: input (str): the key to a G3Timestream map of the source output (str): key of G3Timestream map of output data. if None, input will be overwritten with output - filter_function (function): function that takes frequency in Hz and + filter_function (function): function that takes frequency in Hz and returns a frequency filter - + TODO: - Get rid of numpy fft functions and get faster / better parallizable - options. + Get rid of numpy fft functions and get faster / better parallizable + options. """ - + def __init__(self, input='signal', output='signal_filtered', filter_function=None): if filter_function is None: raise ValueError('Missing Filter Definition') self.filter_function = filter_function super().__init__(input, output) - + def process(self, data, det_name): """ Args: @@ -48,7 +48,7 @@ def process(self, data, det_name): class LowPassButterworth(Filter): """ G3Module for a LowPassButterworth filter - + Attributes: order (int): order of butterworth fc (float): cutoff frequency in Hertz @@ -60,7 +60,7 @@ def __init__(self, input='signal', output='signal_filtered', self.fc=fc self.gain=gain super().__init__(input, output, self.filter_function) - + def filter_function(self, freqs): b, a = signal.butter(self.order, 2*np.pi*self.fc, 'lowpass', analog=True) - return self.gain*np.abs(signal.freqs(b, a, 2*np.pi*freqs)[1]) \ No newline at end of file + return self.gain*np.abs(signal.freqs(b, a, 2*np.pi*freqs)[1]) diff --git a/sotodlib/data/sim.py b/sotodlib/g3_sim.py similarity index 91% rename from sotodlib/data/sim.py rename to sotodlib/g3_sim.py index 251acdfaa..ecd3f7c44 100644 --- a/sotodlib/data/sim.py +++ b/sotodlib/g3_sim.py @@ -9,7 +9,7 @@ import numpy as np from spt3g import core -from sotodlib.data import DataG3Module +from sotodlib.g3_core import DataG3Module class PipelineSeeder(list): @@ -25,17 +25,17 @@ def __call__(self, frame_in): if len(self): output.append(self.pop(0)) return output - + def enumerate_det_id(n_dets, freq=39, band='LF1'): """ Generator for looping through detector names. Not very smart, only for basic testing. - + Args: n_dets (int): number of detector names to make freq (int): detector freqency band (str): detector band - + Returns: SO formatted detector keys """ @@ -45,12 +45,12 @@ def enumerate_det_id(n_dets, freq=39, band='LF1'): t = int(np.mod(n,2)) yield '{}_{:03}_{}_{}'.format(freq,dnum,band,types[t]) -def noise_scan_frames(n_frames=3, n_dets=20, input='signal', n_samps=200, - samp_rate=0.005*core.G3Units.second, +def noise_scan_frames(n_frames=3, n_dets=20, input='signal', n_samps=200, + samp_rate=0.005*core.G3Units.second, t_start=core.G3Time('2020-1-1T00:00:00')): """ - Generate a list of frames filled with noise data and nothing else. - + Generate a list of frames filled with noise data and nothing else. + Args: n_frames (int): number of frames to make n_dets (int): number of detectors per frame @@ -73,17 +73,17 @@ def noise_scan_frames(n_frames=3, n_dets=20, input='signal', n_samps=200, f[input] = tsm t_start += n_samps*samp_rate frame_list.append(f) - return frame_list - + return frame_list + class MakeNoiseData(DataG3Module): """ - Writes a signal with just noise. To be used where an observation - has already been set up but there's no data (such as the output of + Writes a signal with just noise. To be used where an observation + has already been set up but there's no data (such as the output of so3g.python.quicksim.py) mostly just an easy way to get numbers in G3Timestreams - - The noise is a basic white noise plus a 1/f component described by a + + The noise is a basic white noise plus a 1/f component described by a knee frequency and index - + Args: input (str): the key to a G3Timestream map in the G3Frame to replace with noise data @@ -92,17 +92,17 @@ class MakeNoiseData(DataG3Module): white_noise (float): while noise level f_knee (float): knee frequency f_knee_index (float): index of 1/f spectrum, should be negative - + Returns: None """ - def __init__(self, input='signal', output=None, white_noise = 0.005, + def __init__(self, input='signal', output=None, white_noise = 0.005, f_knee = 0.01, f_knee_index=-2): self.white_noise = white_noise self.f_knee = f_knee self.f_knee_index = f_knee_index super().__init__(input, output) - + def process(self, data, k): freqs = np.fft.fftfreq(data.n_samples, core.G3Units.Hz/data.sample_rate) noise = self.white_noise*(1 + (freqs[1:]/self.f_knee)**self.f_knee_index) @@ -110,39 +110,39 @@ def process(self, data, k): ## prevent divide by zero error and assume 1/f doesn't go to infinity noise = np.append(noise[0], noise) return np.real(np.fft.fft(noise)).astype('float64') - + class MakeJumps(DataG3Module): - + """ - G3Module that takes a G3Timestream map and adds randomly + G3Module that takes a G3Timestream map and adds randomly distributed jumps - + Args: input (str): the key to a G3TimestreamMap that is the data source output (str or None): the key for a G3TimestreamMap that will have data plus glitches. If None: jumps are added to Input - info (str): a G3Timestream map will be made with this name that - includes just the jumps. - max_jumps (int): number of jumps in each G3Timestream is + info (str): a G3Timestream map will be made with this name that + includes just the jumps. + max_jumps (int): number of jumps in each G3Timestream is np.random.randint(max_jumps) - height_std_sigma (float): hight of each jump is a draw from a normal + height_std_sigma (float): hight of each jump is a draw from a normal distribution with standard deviation of height_std_sigma*np.std(timestream) """ - - def __init__(self, input='signal', output=None, info='flags_encoded_jumps', + + def __init__(self, input='signal', output=None, info='flags_encoded_jumps', max_jumps=3, height_std_sigma=10): self.info = info self.max_jumps = max_jumps self.height_std_sigma = height_std_sigma super().__init__(input, output) - + def __call__(self, f): if f.type == core.G3FrameType.Scan: self.jump_map = core.G3TimestreamMap() super().__call__(f) - + if f.type == core.G3FrameType.Scan: self.jump_map.start = f[self.input].start self.jump_map.stop = f[self.input].stop @@ -154,51 +154,51 @@ def process(self, data, det_name): jumps = np.zeros( (data.n_samples,) ) for i in range(len(locs)): jumps[locs[i]:] += heights[i] - + self.jump_map[det_name] = core.G3Timestream( jumps ) return data + jumps - + class MakeGlitches(DataG3Module): """ - G3Module that takes the G3Timestream map and adds randomly + G3Module that takes the G3Timestream map and adds randomly distributed glitches - + Args: input (str): the key to a G3TimestreamMap that is the data source output (str or None): the key for a G3TimestreamMap that will have data plus glitches. If None: Glitches are added to Input - info (str): a G3Timestream map will be made with this name that - includes just the glitches. - max_glitches (int): number of glitches in each G3Timestream is + info (str): a G3Timestream map will be made with this name that + includes just the glitches. + max_glitches (int): number of glitches in each G3Timestream is np.random.randint(max_glitches) - height_std_sigma (float): hight of each jump is a draw from a normal + height_std_sigma (float): hight of each jump is a draw from a normal distribution with standard deviation of height_std_sigma*np.std(timestream) """ - + def __init__(self, input='signal', output=None, info='flags_encoded_glitches', max_glitches=3, height_std_sigma=20): self.info = info self.max_glitches = max_glitches self.height_std_sigma = height_std_sigma super().__init__(input, output) - + def __call__(self, f): if f.type == core.G3FrameType.Scan: self.glitch_map = core.G3TimestreamMap() super().__call__(f) - + if f.type == core.G3FrameType.Scan: self.glitch_map.start = f[self.input].start self.glitch_map.stop = f[self.input].stop f[self.info] = self.glitch_map - + def process(self, data, det_name): locs = np.random.randint(data.n_samples, size=(np.random.randint(self.max_glitches),) ) heights = np.random.randn( len(locs) )*self.height_std_sigma*np.std(data) glitches = np.zeros( (data.n_samples,) ) glitches[locs] += heights - + self.glitch_map[det_name] = core.G3Timestream( glitches ) - return core.G3Timestream( data + glitches ) \ No newline at end of file + return core.G3Timestream( data + glitches ) diff --git a/sotodlib/hardware/__init__.py b/sotodlib/hardware/__init__.py deleted file mode 100644 index 7f574ccb6..000000000 --- a/sotodlib/hardware/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2018-2019 Simons Observatory. -# Full license can be found in the top level "LICENSE" file. -"""Hardware models for use in analysis. - -This module contains code for simulating a hardware model and -dumping / loading hardware information to / from disk. - -""" - -# These are simply namespace imports for convenience. - -from .config import (Hardware, get_example) - -from .sim import (sim_wafer_detectors, sim_telescope_detectors) - -from .vis import (plot_detectors, summary_text) diff --git a/sotodlib/hardware/db.py b/sotodlib/hardware/db.py deleted file mode 100644 index 2ea7b3d3b..000000000 --- a/sotodlib/hardware/db.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright (c) 2018-2019 Simons Observatory. -# Full license can be found in the top level "LICENSE" file. -"""Tools for working with hardware databases. -""" - -import os - -from collections import OrderedDict - -from contextlib import contextmanager - -import numpy as np - -import sqlite3 - -#from sotoddb import DetDB - - -# NOTE: This source is not currently used. It is kept here in case we revisit -# the use of databases. - - -class DataBase(object): - """Class representing a hardware configuration database. - - This class provides an interface to an sqlite DB supported by the - sotoddb package. In addition to detector property tables, other tables - are used to represent hardware features. - - Args: - path (str, optional): Path to the DB file. If the file does not - exist, it is created and opened in write mode. If it does exist, - it is opened in the mode specified. - mode (str, optional): The mode ("r" or "w") to use when opening the - DB. If not specified, the default is "r" if the file exists - else "w" if it is being created. In-memory DBs are always "w". - conf (dict, optional): If the database is being created, this config - dictionary is used to create auxilliary tables. - dets (dict, optional): If the database is being created, this detector - dictionary is used to create the detector property tables. - - """ - def __init__(self, path, mode=None, conf=None, dets=None): - self._path = path - self._mode = mode - - create = True - if os.path.exists(self._path): - create = False - - if self._mode == 'r' and create: - raise RuntimeError("cannot open a non-existent DB in read-only " - " mode") - - self._connstr = None - - # This timeout is in seconds - self._busytime = 1000 - - # Journaling options - self._journalmode = "persist" - self._syncmode = "normal" - - if create: - self.initdb(conf, dets) - return - - def _open(self): - try: - # only python3 supports uri option - if self._mode == 'r': - self._connstr = 'file:{}?mode=ro'.format(self._path) - else: - self._connstr = 'file:{}?mode=rwc'.format(self._path) - self._conn = sqlite3.connect(self._connstr, uri=True, - timeout=self._busytime) - except sqlite3.OperationalError: - self._conn = sqlite3.connect(self._path, - timeout=self._busytime) - if self._mode == 'w': - # In read-write mode, set the journaling - self._conn.execute("pragma journal_mode={}" - .format(self._journalmode)) - self._conn.execute("pragma synchronous={}".format(self._syncmode)) - # Other tuning options - self._conn.execute("pragma temp_store=memory") - self._conn.execute("pragma page_size=4096") - self._conn.execute("pragma cache_size=4000") - return - - def _close(self): - del self._conn - self._conn = None - return - - @contextmanager - def cursor(self): - self._open() - cur = self._conn.cursor() - cur.execute("begin transaction") - try: - yield cur - except sqlite3.DatabaseError as err: - cur.execute("rollback") - raise err - else: - try: - cur.execute("commit") - except sqlite3.OperationalError: - # sqlite3 in py3.5 can't commit a read-only finished - # transaction. - pass - finally: - del cur - self._close() - - - def initdb(self, conf, dets): - """Initialize the database. - - We use sotoddb to create the database and populate the detector - tables. Then we directly create the other tables. - - Args: - conf (dict): The hardware config dictionary used to create - auxilliary tables. - dets (dict): The detector dictionary used to create the detector - property tables. - - """ - # Get the main detector schema from the first entry - dkeys = list(dets.keys()) - dprops = dets[dkeys[0]] - dschema = [ - "`det_id` integer", - "`time0` integer", - "`time1` integer", - "`name` text" - ] - for k, v in dprops.items(): - if k == "quat": - # We keep the pointing offsets in a separate table. - continue - coltype = "text" - if isinstance(v, float): - coltype = "float" - elif isinstance(v, int): - coltype = "integer" - colstr = "`{}` {}".format(k, coltype) - dschema.append(colstr) - dqschema = [ - "`det_id` integer", - "`time0` integer", - "`time1` integer", - "`qx` float", - "`qy` float", - "`qz` float", - "`qw` float" - ] - - # Create the DB and detector tables, then close. - sodb = DetDB(map_file=self._path, init_db=True) - sodb.create_table("detprops", dschema) - sodb.create_table("detgeom", dqschema) - del sodb - - # Now go and create our other tables and populate everything. - - # Go through the hardware config and grab the first row of each - # dictionary to create the schema. - for table, props in conf.items(): - pkeys = list(props.keys()) - row = pkeys[0] - colprops = props[row] - createstr = "create table {} (name text unique".format(table) - for k, v in colprops.items(): - coltype = "text" - if isinstance(v, float): - coltype = "float" - elif isinstance(v, int): - coltype = "integer" - createstr = "{}, {} {}".format(createstr, k, coltype) - createstr = "{})".format(createstr) - print(createstr, flush=True) - with self.cursor() as cur: - cur.execute(createstr) - - # Now go back and populate the hardware config tables. One transaction - # per table. - for table, props in conf.items(): - with self.cursor() as cur: - for name, prp in props.items(): - colstr = "(name" - valstr = "('{}'".format(name) - for k, v in prp.items(): - colstr += ", {}".format(k) - if isinstance(v, (float, int)): - valstr += ", {}".format(v) - else: - valstr += ", '{}'".format(v) - colstr += ")" - valstr += ")" - com = "insert into {} {} values {}".format( - table, colstr, valstr) - print(com, flush=True) - cur.execute(com) - - # Now populate the detector tables - with self.cursor() as cur: - for det, props in dets.items(): - detid = props["ID"] - pcol = "(det_id, name" - pval = "({}, '{}'".format(detid, det) - gcol = "(det_id, qx, qy, qz, qw)" - gval = "({}".format(detid) - for k, v in props.items(): - if k == "quat": - for i in range(4): - gval += ", {}".format(v[i]) - gval += ")" - else: - pcol += ", {}".format(k) - if isinstance(v, (float, int)): - pval += ", {}".format(v) - else: - pval += ", '{}'".format(v) - pcol += ")" - pval += ")" - com = "insert into detprops {} values {}".format(pcol, pval) - print(com, flush=True) - cur.execute(com) - com = "insert into detgeom {} values {}".format(gcol, gval) - print(com, flush=True) - cur.execute(com) - return diff --git a/sotodlib/io/__init__.py b/sotodlib/io/__init__.py new file mode 100644 index 000000000..4fa450028 --- /dev/null +++ b/sotodlib/io/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2018-2020 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +"""Simons Observatory I/O routines. + +This module contains code for reading and writing data and metadata. + +""" diff --git a/sotodlib/data/load.py b/sotodlib/io/load.py similarity index 98% rename from sotodlib/data/load.py rename to sotodlib/io/load.py index 90f1579e0..c25a99f42 100644 --- a/sotodlib/data/load.py +++ b/sotodlib/io/load.py @@ -6,9 +6,9 @@ The basic routines here provide structures for automatically unpacking series of G3 frames into coherent data structures. -Additional routines use an ObsFileDB (provided by the user) to +Additional routines use an ObsFileDb (provided by the user) to optimize disk reads for loading particular data of interest to a user. -These routines rely on the completeness and accuracy of the ObsFileDB, +These routines rely on the completeness and accuracy of the ObsFileDb, and are appropriate for large archives of files that have been pre-scanned to log their contents. @@ -355,14 +355,14 @@ def load_observation(db, obs_id, dets=None, prefix=None): Arguments: - db (ObsFileDB): The database describing this observation file + db (ObsFileDb): The database describing this observation file set. obs_id (str): The identifier of the observation. dets (list of str): The detector names of interest. If None, loads all dets present in this observation. To load only the ancillary data, pass an empty list. prefix (str): The root address of the data files. If not - specified, the prefix is taken from the ObsFileDB. + specified, the prefix is taken from the ObsFileDb. Returns an AxisManager with the data. @@ -489,9 +489,9 @@ def hstack_into(dest, src_arrays): #: #: loader(db, obs_id, dets=None, prefix=None) #: -#: Here db is an ObsFileDB, obs_id is a string, dets is a list of +#: Here db is an ObsFileDb, obs_id is a string, dets is a list of #: string names of readout channels, and prefix is a string that overrides -#: the path prefix of ObsFileDB. +#: the path prefix of ObsFileDb. #: #: "This is an interim solution and the API will change", he said in #: March 2020. diff --git a/sotodlib/io/metadata.py b/sotodlib/io/metadata.py new file mode 100644 index 000000000..9bb6d7bac --- /dev/null +++ b/sotodlib/io/metadata.py @@ -0,0 +1,7 @@ +# Copyright (c) 2018-2020 Simons Observatory. +# Full license can be found in the top level "LICENSE" file. +"""Metadata I/O. +""" + +from sotoddb import simple, loader +from sotoddb import SuperLoader diff --git a/sotodlib/data/toast_export.py b/sotodlib/io/toast_export.py similarity index 100% rename from sotodlib/data/toast_export.py rename to sotodlib/io/toast_export.py diff --git a/sotodlib/data/toast_frame_utils.py b/sotodlib/io/toast_frame_utils.py similarity index 100% rename from sotodlib/data/toast_frame_utils.py rename to sotodlib/io/toast_frame_utils.py diff --git a/sotodlib/data/toast_load.py b/sotodlib/io/toast_load.py similarity index 100% rename from sotodlib/data/toast_load.py rename to sotodlib/io/toast_load.py diff --git a/sotodlib/metadata/__init__.py b/sotodlib/metadata/__init__.py deleted file mode 100644 index 1ccbf65d4..000000000 --- a/sotodlib/metadata/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# The metadata engine. Supports a number of classes for holding (and -# reading and writing) fairly generic metadata. -# -# Currently we just import all from the sotoddb library -- this latter -# should be merged into sotodlib soon. -# - -from sotoddb import * diff --git a/sotodlib/modules/__init__.py b/sotodlib/modules/__init__.py deleted file mode 100644 index 61594d07b..000000000 --- a/sotodlib/modules/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#Add your modules to the package here diff --git a/sotodlib/observe/__init__.py b/sotodlib/observe/__init__.py deleted file mode 100644 index 84436f957..000000000 --- a/sotodlib/observe/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2018-2019 Simons Observatory. -# Full license can be found in the top level "LICENSE" file. -"""Tools for simulating and working with observation / scanning information. -""" diff --git a/sotodlib/observe/sim.py b/sotodlib/observe/sim.py deleted file mode 100644 index d3fbaf58e..000000000 --- a/sotodlib/observe/sim.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2018-2019 Simons Observatory. -# Full license can be found in the top level "LICENSE" file. -"""Tools for simulating observations. -""" diff --git a/sotodlib/scripts/__init__.py b/sotodlib/scripts/__init__.py index a0aa9d7c0..1e5d42713 100644 --- a/sotodlib/scripts/__init__.py +++ b/sotodlib/scripts/__init__.py @@ -1,6 +1,6 @@ -# Copyright (c) 2018 Simons Observatory. +# Copyright (c) 2018-2020 Simons Observatory. # Full license can be found in the top level "LICENSE" file. -"""Simons Observatory TOD processing entry points. +"""Simons Observatory script entry points. This module contains files which have a main() function defined, and which are defined as entry points in setup.py. diff --git a/sotodlib/scripts/hardware_info.py b/sotodlib/scripts/hardware_info.py index 52260e953..c122cbbf4 100644 --- a/sotodlib/scripts/hardware_info.py +++ b/sotodlib/scripts/hardware_info.py @@ -5,7 +5,9 @@ import argparse -from ..hardware import Hardware, summary_text +from ..core import Hardware + +from ..vis_hardware import summary_text def main(): diff --git a/sotodlib/scripts/hardware_plot.py b/sotodlib/scripts/hardware_plot.py index cc3006a9d..b0e9ea229 100644 --- a/sotodlib/scripts/hardware_plot.py +++ b/sotodlib/scripts/hardware_plot.py @@ -5,7 +5,9 @@ import argparse -from ..hardware import Hardware, plot_detectors, summary_text +from ..core import Hardware + +from ..vis_hardware import plot_detectors, summary_text def main(): diff --git a/sotodlib/scripts/hardware_sim.py b/sotodlib/scripts/hardware_sim.py index d6c57f668..dd697974a 100644 --- a/sotodlib/scripts/hardware_sim.py +++ b/sotodlib/scripts/hardware_sim.py @@ -8,7 +8,7 @@ from collections import OrderedDict -from ..hardware import get_example, sim_telescope_detectors +from ..sim_hardware import get_example, sim_telescope_detectors def main(): diff --git a/sotodlib/scripts/hardware_trim.py b/sotodlib/scripts/hardware_trim.py index 7504c0a83..ac6a0df22 100644 --- a/sotodlib/scripts/hardware_trim.py +++ b/sotodlib/scripts/hardware_trim.py @@ -8,7 +8,7 @@ from collections import OrderedDict -from ..hardware import Hardware +from ..core import Hardware def main(): diff --git a/sotodlib/hardware/sim.py b/sotodlib/sim_hardware.py similarity index 71% rename from sotodlib/hardware/sim.py rename to sotodlib/sim_hardware.py index 273006171..7a9cb519c 100644 --- a/sotodlib/hardware/sim.py +++ b/sotodlib/sim_hardware.py @@ -13,6 +13,8 @@ import quaternionarray as qa +from .core import Hardware + # FIXME: much of this code is copy/pasted from the toast source, simply to # avoid a dependency. Once we can "pip install toast", we should consider # just calling that package. @@ -667,3 +669,354 @@ def sim_telescope_detectors(hw, tele, tubes=None): windx += 1 tindx += 1 return alldets + + +def get_example(): + """Return an example Hardware config with the required sections. + + The returned Hardware object has 4 fake detectors as an example. These + detectors can be replaced by the results of other simulation functions. + + Returns: + (Hardware): Hardware object with example parameters. + + """ + cnf = OrderedDict() + + bands = OrderedDict() + + bnd = OrderedDict() + bnd["center"] = 25.7 + bnd["low"] = 21.7 + bnd["high"] = 29.7 + bnd["bandpass"] = "" + bnd["NET"] = 300.0 + bnd["fknee"] = 50.0 + bnd["fmin"] = 0.01 + bnd["alpha"] = 3.5 + # Noise elevation scaling fits from Carlos Sierra + # These numbers are for V3 LAT baseline + bnd["A"] = 0.09 + bnd["C"] = 0.87 + bands["LF1"] = bnd + + bnd = OrderedDict() + bnd["center"] = 38.9 + bnd["low"] = 30.9 + bnd["high"] = 46.9 + bnd["bandpass"] = "" + bnd["NET"] = 300.0 + bnd["fknee"] = 50.0 + bnd["fmin"] = 0.01 + bnd["alpha"] = 3.5 + bnd["A"] = 0.25 + bnd["C"] = 0.64 + bands["LF2"] = bnd + + bnd = OrderedDict() + bnd["center"] = 92.0 + bnd["low"] = 79.0 + bnd["high"] = 105.0 + bnd["bandpass"] = "" + bnd["NET"] = 300.0 + bnd["fknee"] = 50.0 + bnd["fmin"] = 0.01 + bnd["alpha"] = 3.5 + bnd["A"] = 0.14 + bnd["C"] = 0.80 + bands["MFF1"] = bnd + + bnd = OrderedDict() + bnd["center"] = 147.5 + bnd["low"] = 130.0 + bnd["high"] = 165.0 + bnd["bandpass"] = "" + bnd["NET"] = 400.0 + bnd["fknee"] = 50.0 + bnd["fmin"] = 0.01 + bnd["alpha"] = 3.5 + bnd["A"] = 0.17 + bnd["C"] = 0.76 + bands["MFF2"] = bnd + + bnd = OrderedDict() + bnd["center"] = 88.6 + bnd["low"] = 75.6 + bnd["high"] = 101.6 + bnd["bandpass"] = "" + bnd["NET"] = 300.0 + bnd["fknee"] = 50.0 + bnd["fmin"] = 0.01 + bnd["alpha"] = 3.5 + bnd["A"] = 0.19 + bnd["C"] = 0.74 + bands["MFS1"] = bnd + + bnd = OrderedDict() + bnd["center"] = 146.5 + bnd["low"] = 128.0 + bnd["high"] = 165.0 + bnd["bandpass"] = "" + bnd["NET"] = 400.0 + bnd["fknee"] = 50.0 + bnd["fmin"] = 0.01 + bnd["alpha"] = 3.5 + bnd["A"] = 0.19 + bnd["C"] = 0.73 + bands["MFS2"] = bnd + + bnd = OrderedDict() + bnd["center"] = 225.7 + bnd["low"] = 196.7 + bnd["high"] = 254.7 + bnd["bandpass"] = "" + bnd["NET"] = 400.0 + bnd["fknee"] = 50.0 + bnd["fmin"] = 0.01 + bnd["alpha"] = 3.5 + bnd["A"] = 0.30 + bnd["C"] = 0.58 + bands["UHF1"] = bnd + + bnd = OrderedDict() + bnd["center"] = 285.4 + bnd["low"] = 258.4 + bnd["high"] = 312.4 + bnd["bandpass"] = "" + bnd["NET"] = 400.0 + bnd["fknee"] = 50.0 + bnd["fmin"] = 0.01 + bnd["alpha"] = 3.5 + bnd["A"] = 0.36 + bnd["C"] = 0.49 + bands["UHF2"] = bnd + + cnf["bands"] = bands + + wafers = OrderedDict() + + wtypes = ["UHF", "MFF", "MFS", "LF"] + wcnt = { + "LF": 1*7 + 1*3, + "MFF": 1*7 + 2*3, + "MFS": 1*7 + 2*3, + "UHF": 1*7 + 2*3 + } + wnp = { + "LF": 37, + "MFF": 432, + "MFS": 397, + "UHF": 432 + } + wpixmm = { + "LF": 18.0, + "MFF": 5.3, + "MFS": 5.6, + "UHF": 5.3 + } + wrhombgap = { + "MFF": 0.71, + "UHF": 0.71, + } + wbd = { + "LF": ["LF1", "LF2"], + "MFF": ["MFF1", "MFF2"], + "MFS": ["MFS1", "MFS2"], + "UHF": ["UHF1", "UHF2"] + } + windx = 0 + cardindx = 0 + for wt in wtypes: + for ct in range(wcnt[wt]): + wn = "{:02d}".format(windx) + wf = OrderedDict() + wf["type"] = wt + if (wt == "LF") or (wt == "MFS"): + wf["packing"] = "S" + else: + wf["packing"] = "F" + wf["rhombusgap"] = wrhombgap[wt] + wf["npixel"] = wnp[wt] + wf["pixsize"] = wpixmm[wt] + wf["bands"] = wbd[wt] + wf["card"] = "{:02d}".format(cardindx) + cardindx += 1 + wafers[wn] = wf + windx += 1 + + cnf["wafers"] = wafers + + tubes = OrderedDict() + + woff = { + "LF": 0, + "MFF": 0, + "MFS": 0, + "UHF": 0 + } + + ltubes = ["UHF", "UHF", "MFF", "MFF", "MFS", "MFS", "LF"] + ltubepos = [0, 1, 2, 3, 5, 6, 10] + for tindx in range(7): + nm = "LT{:d}".format(tindx) + ttyp = ltubes[tindx] + tb = OrderedDict() + tb["type"] = ttyp + tb["waferspace"] = 127.89 + tb["wafers"] = list() + for tw in range(3): + off = 0 + for w, props in cnf["wafers"].items(): + if props["type"] == ttyp: + if off == woff[ttyp]: + tb["wafers"].append(w) + woff[ttyp] += 1 + break + off += 1 + tb["location"] = ltubepos[tindx] + tubes[nm] = tb + + stubes = ["UHF", "MFF", "MFS", "LF"] + for tindx in range(4): + nm = "ST{:d}".format(tindx) + ttyp = stubes[tindx] + tb = OrderedDict() + tb["type"] = ttyp + tb["waferspace"] = 127.89 + tb["wafers"] = list() + for tw in range(7): + off = 0 + for w, props in cnf["wafers"].items(): + if props["type"] == ttyp: + if off == woff[ttyp]: + tb["wafers"].append(w) + woff[ttyp] += 1 + break + off += 1 + tb["location"] = 0 + tubes[nm] = tb + + cnf["tubes"] = tubes + + telescopes = OrderedDict() + + tele = OrderedDict() + tele["tubes"] = ["LT0", "LT1", "LT2", "LT3", "LT4", "LT5", "LT6"] + tele["platescale"] = 0.00495 + # This tube spacing in mm corresponds to 1.78 degrees projected on + # the sky at a plate scale of 0.00495 deg/mm. + tele["tubespace"] = 359.6 + fwhm = OrderedDict() + fwhm["LF1"] = 7.4 + fwhm["LF2"] = 5.1 + fwhm["MFF1"] = 2.2 + fwhm["MFF2"] = 1.4 + fwhm["MFS1"] = 2.2 + fwhm["MFS2"] = 1.4 + fwhm["UHF1"] = 1.0 + fwhm["UHF2"] = 0.9 + tele["fwhm"] = fwhm + telescopes["LAT"] = tele + + sfwhm = OrderedDict() + scale = 0.09668 / 0.00495 + for k, v in fwhm.items(): + sfwhm[k] = float(int(scale * v * 10.0) // 10) + + tele = OrderedDict() + tele["tubes"] = ["ST0"] + tele["platescale"] = 0.09668 + tele["fwhm"] = sfwhm + telescopes["SAT0"] = tele + + tele = OrderedDict() + tele["tubes"] = ["ST1"] + tele["platescale"] = 0.09668 + tele["fwhm"] = sfwhm + telescopes["SAT1"] = tele + + tele = OrderedDict() + tele["tubes"] = ["ST2"] + tele["platescale"] = 0.09668 + tele["fwhm"] = sfwhm + telescopes["SAT2"] = tele + + tele = OrderedDict() + tele["tubes"] = ["ST3"] + tele["platescale"] = 0.09668 + tele["fwhm"] = sfwhm + telescopes["SAT3"] = tele + + cnf["telescopes"] = telescopes + + cards = OrderedDict() + crates = OrderedDict() + + crt_indx = 0 + + for tel in cnf["telescopes"]: + crn = "{:d}".format(crt_indx) + crt = OrderedDict() + crt["cards"] = list() + crt["telescope"] = tel + + ## get all the wafer card numbers for a telescope + tb_wfrs = [cnf["tubes"][t]["wafers"] for t in cnf["telescopes"][tel]["tubes"]] + tl_wfrs = [i for sl in tb_wfrs for i in sl] + wafer_cards = [cnf["wafers"][w]["card"] for w in tl_wfrs] + + # add all cards to the card table and assign to crates + for crd in wafer_cards: + cdprops = OrderedDict() + cdprops["nbias"] = 12 + cdprops["ncoax"] = 2 + cdprops["nchannel"] = 2000 + cards[crd] = cdprops + + crt["cards"].append(crd) + + # name new crates when current one is full + if ('S' in tel and len(crt["cards"]) >=4) or len(crt["cards"]) >=6: + crates[crn] = crt + crt_indx += 1 + crn = "{:d}".format(crt_indx) + crt = OrderedDict() + crt["cards"] = list() + crt["telescope"] = tel + + # each telescope starts with a new crate + crates[crn] = crt + crt_indx += 1 + + cnf["cards"] = cards + cnf["crates"] = crates + + pl = ["A", "B"] + hand = ["L", "R"] + + dets = OrderedDict() + for d in range(4): + dprops = OrderedDict() + dprops["wafer"] = "42" + dprops["ID"] = d + dprops["pixel"] = "000" + bindx = d % 2 + dprops["band"] = "LF{}".format(bindx) + dprops["fwhm"] = 1.0 + dprops["pol"] = pl[bindx] + dprops["handed"] = hand[bindx] + dprops["card"] = "42" + dprops["channel"] = d + dprops["coax"] = 0 + dprops["bias"] = 0 + dprops["quat"] = np.array([0.0, 0.0, 0.0, 1.0]) + dname = "{}_{}_{}_{}".format("42", "000", dprops["band"], + dprops["pol"]) + dets[dname] = dprops + + cnf["detectors"] = dets + + hw = Hardware() + hw.data = cnf + + return hw diff --git a/sotodlib/pipeline_tools/__init__.py b/sotodlib/utils/pipeline_tools/__init__.py similarity index 81% rename from sotodlib/pipeline_tools/__init__.py rename to sotodlib/utils/pipeline_tools/__init__.py index 82abbf9af..eafc97209 100644 --- a/sotodlib/pipeline_tools/__init__.py +++ b/sotodlib/utils/pipeline_tools/__init__.py @@ -1,5 +1,8 @@ -# Copyright (c) 2019 Simons Observatory. +# Copyright (c) 2018-2020 Simons Observatory. # Full license can be found in the top level "LICENSE" file. +""" +Utilities for building toast pipelines. +""" from .atm import scale_atmosphere_by_bandpass from .export import add_export_args, export_TOD diff --git a/sotodlib/pipeline_tools/atm.py b/sotodlib/utils/pipeline_tools/atm.py similarity index 100% rename from sotodlib/pipeline_tools/atm.py rename to sotodlib/utils/pipeline_tools/atm.py diff --git a/sotodlib/pipeline_tools/export.py b/sotodlib/utils/pipeline_tools/export.py similarity index 100% rename from sotodlib/pipeline_tools/export.py rename to sotodlib/utils/pipeline_tools/export.py diff --git a/sotodlib/pipeline_tools/hardware.py b/sotodlib/utils/pipeline_tools/hardware.py similarity index 100% rename from sotodlib/pipeline_tools/hardware.py rename to sotodlib/utils/pipeline_tools/hardware.py diff --git a/sotodlib/pipeline_tools/noise.py b/sotodlib/utils/pipeline_tools/noise.py similarity index 100% rename from sotodlib/pipeline_tools/noise.py rename to sotodlib/utils/pipeline_tools/noise.py diff --git a/sotodlib/pipeline_tools/observation.py b/sotodlib/utils/pipeline_tools/observation.py similarity index 100% rename from sotodlib/pipeline_tools/observation.py rename to sotodlib/utils/pipeline_tools/observation.py diff --git a/sotodlib/pipeline_tools/pysm.py b/sotodlib/utils/pipeline_tools/pysm.py similarity index 100% rename from sotodlib/pipeline_tools/pysm.py rename to sotodlib/utils/pipeline_tools/pysm.py diff --git a/sotodlib/hardware/vis.py b/sotodlib/vis_hardware.py similarity index 100% rename from sotodlib/hardware/vis.py rename to sotodlib/vis_hardware.py diff --git a/tests/test_data.py b/tests/test_data.py index fe5482d8a..eb52db816 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -9,29 +9,29 @@ import numpy as np from spt3g import core -from sotodlib.data import DataG3Module -from sotodlib.data.filter import Filter, LowPassButterworth -from sotodlib.data.condition import (Detrend, Retrend, MeanSubtract, +from sotodlib.g3_core import DataG3Module +from sotodlib.g3_filter import Filter, LowPassButterworth +from sotodlib.g3_condition import (Detrend, Retrend, MeanSubtract, MedianSubtract, Decimate, Resample) -import sotodlib.data.sim as data_sim +import sotodlib.g3_sim as data_sim class DataTest(TestCase): - + def setUp(self): self.frames = data_sim.noise_scan_frames(input='signal') - + def test_DataG3Module(self): ### Test that it works in a pipeline p = core.G3Pipeline() p.Add(data_sim.PipelineSeeder(self.frames)) p.Add(DataG3Module, input='signal', output=None) p.Run() - + ### test that it works on individual frames x = DataG3Module(input='signal', output=None) x.apply(self.frames[0]) - + ### Test that it works as an inline function p = core.G3Pipeline() p.Add(data_sim.PipelineSeeder(self.frames)) @@ -42,39 +42,39 @@ def test_filters(self): ### Test general filter p = core.G3Pipeline() p.Add(data_sim.PipelineSeeder(self.frames)) - p.Add(Filter, input='signal', output=None, + p.Add(Filter, input='signal', output=None, filter_function=lambda freqs:np.ones_like(freqs)) p.Run() - + p = core.G3Pipeline() p.Add(data_sim.PipelineSeeder(self.frames)) p.Add(LowPassButterworth) p.Run() - + def test_condition(self): p = core.G3Pipeline() p.Add(data_sim.PipelineSeeder(self.frames)) p.Add(Detrend,input='signal', output=None) p.Add(Retrend,input='signal', output=None) p.Run() - + p = core.G3Pipeline() p.Add(data_sim.PipelineSeeder(self.frames)) p.Add(MeanSubtract,input='signal', output=None) p.Add(MedianSubtract,input='signal', output=None) p.Run() - + p = core.G3Pipeline() p.Add(data_sim.PipelineSeeder(self.frames)) p.Add(Decimate,input='signal', output=None) p.Run() - + p = core.G3Pipeline() p.Add(data_sim.PipelineSeeder(self.frames)) p.Add(Resample,input='signal', output=None) p.Run() - - - + + + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_hardware.py b/tests/test_hardware.py index 72bca2c59..8284c1cc5 100644 --- a/tests/test_hardware.py +++ b/tests/test_hardware.py @@ -11,12 +11,14 @@ from ._helpers import create_outdir -from sotodlib.hardware.config import Hardware, get_example +from sotodlib.core import Hardware -from sotodlib.hardware.sim import (sim_wafer_detectors, +from sotodlib.sim_hardware import get_example + +from sotodlib.sim_hardware import (sim_wafer_detectors, sim_telescope_detectors) -from sotodlib.hardware.vis import plot_detectors +from sotodlib.vis_hardware import plot_detectors class HardwareTest(TestCase): diff --git a/tests/test_toast_export.py b/tests/test_toast_export.py index c5100f838..c642098bd 100644 --- a/tests/test_toast_export.py +++ b/tests/test_toast_export.py @@ -11,9 +11,9 @@ from ._helpers import create_outdir -from sotodlib.hardware.config import get_example +from sotodlib.sim_hardware import get_example -from sotodlib.hardware.sim import sim_telescope_detectors +from sotodlib.sim_hardware import sim_telescope_detectors # Import so3g first so that it can control the import and monkey-patching # of spt3g. Then our import of spt3g_core will use whatever has been imported @@ -29,7 +29,7 @@ from toast.mpi import MPI from toast.tod import TODGround from toast.tod import AnalyticNoise - from sotodlib.data.toast_export import ToastExport + from sotodlib.io.toast_export import ToastExport toast_available = True except ImportError: toast_available = False diff --git a/tests/test_toast_load.py b/tests/test_toast_load.py index 12086a48d..ba6a4e86b 100644 --- a/tests/test_toast_load.py +++ b/tests/test_toast_load.py @@ -15,9 +15,9 @@ from ._helpers import create_outdir -from sotodlib.hardware.config import get_example +from sotodlib.sim_hardware import get_example -from sotodlib.hardware.sim import sim_telescope_detectors +from sotodlib.sim_hardware import sim_telescope_detectors # Import so3g first so that it can control the import and monkey-patching # of spt3g. Then our import of spt3g_core will use whatever has been imported @@ -33,8 +33,8 @@ from toast.mpi import MPI from toast.tod import TODGround from toast.tod import AnalyticNoise - from sotodlib.data.toast_export import ToastExport - from sotodlib.data.toast_load import load_data + from sotodlib.io.toast_export import ToastExport + from sotodlib.io.toast_load import load_data toast_available = True except ImportError: toast_available = False