diff --git a/.coveragerc b/.coveragerc index 91fbecdc..526c407d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,8 +2,10 @@ omit = */tests/* */data/* - + */scripts/* + [report] omit = */tests/* */data/* + */scripts/* \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 64ade660..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,12 +0,0 @@ -include *.md -include LICENSE -include hera_qm/VERSION -include hera_qm/GIT_INFO -include *.bib -include scripts/*.pro -include scripts/*.sh -recursive-include hera_qm/data * -prune hera_qm/data/test -recursive-include docs * -exclude docs/.DS_Store -prune docs/_build diff --git a/scripts/ant_metrics_example_notebook.ipynb b/hera_qm/scripts/ant_metrics_example_notebook.ipynb similarity index 100% rename from scripts/ant_metrics_example_notebook.ipynb rename to hera_qm/scripts/ant_metrics_example_notebook.ipynb diff --git a/hera_qm/scripts/ant_metrics_run.py b/hera_qm/scripts/ant_metrics_run.py new file mode 100755 index 00000000..33e3337d --- /dev/null +++ b/hera_qm/scripts/ant_metrics_run.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +from hera_qm import utils +from hera_qm import ant_metrics +import sys + +def main(): + ap = utils.get_metrics_ArgumentParser('ant_metrics') + args = ap.parse_args() + history = ' '.join(sys.argv) + ant_metrics.ant_metrics_run(args.sum_files, + diff_files=args.diff_files, + apriori_xants=args.apriori_xants, + a_priori_xants_yaml=args.a_priori_xants_yaml, + crossCut=args.crossCut, + deadCut=args.deadCut, + metrics_path=args.metrics_path, + extension=args.extension, + overwrite=args.clobber, + verbose=args.verbose, + history=history) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/hera_qm/scripts/auto_metrics_run.py b/hera_qm/scripts/auto_metrics_run.py new file mode 100755 index 00000000..98ec055f --- /dev/null +++ b/hera_qm/scripts/auto_metrics_run.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2021 the HERA Project +# Licensed under the MIT License + +from hera_qm import utils +from hera_qm.auto_metrics import auto_metrics_run +import sys + +def main(): + ap = utils.get_metrics_ArgumentParser('auto_metrics') + args = ap.parse_args() + history = ' '.join(sys.argv) + + auto_metrics_run(args.metric_outfile, + args.raw_auto_files, + median_round_modz_cut=args.median_round_modz_cut, + mean_round_modz_cut=args.mean_round_modz_cut, + edge_cut=args.edge_cut, + Kt=args.Kt, + Kf=args.Kf, + sig_init=args.sig_init, + sig_adj=args.sig_adj, + chan_thresh_frac=args.chan_thresh_frac, + history=history, + overwrite=args.clobber) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/hera_qm/scripts/auto_view.py b/hera_qm/scripts/auto_view.py new file mode 100755 index 00000000..5e4b8970 --- /dev/null +++ b/hera_qm/scripts/auto_view.py @@ -0,0 +1,340 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +import os +import argparse +import numpy as np +import re +import matplotlib as mpl +import redis as redis_lib +from hera_mc import mc, sys_handling, cm_utils +from astropy import time +from matplotlib.colors import LogNorm +from matplotlib.colors import LinearSegmentedColormap +import pyuvdata.utils as uvutils +import aipy.miriad as apm + +def main(): + parser = argparse.ArgumentParser(description='Plot auto locations and magnitudes') + parser.add_argument('-s', '--show', dest='show', action='store_true', default=False, + help='Show the plot. Default: False -- i.e., just save a png') + parser.add_argument('-l', '--log', dest='log', action='store_true', default=True, + help='Take 10*log10() of data before plotting. Default:True') + parser.add_argument('--outpath', default='', type=str, + help='Path to save output plots to. Default is same directory as file.') + parser.add_argument('--outbase', default='', type=str, + help='Base for output file names. Default is JD of data.') + parser.add_argument('--idbaddies', dest='idbaddies', action='store_true', default=False, + help='Identify potential misbehaving antennas. Default: False') + parser.add_argument('files', metavar='files', type=str, nargs='*', default=[], + help='Files for which to plot auto views.') + args = parser.parse_args() + + if not args.show: + # use a matplotlib backend that doesn't require an X session + mpl.use('Agg') + # this has to be called after matplotlibs use() + import matplotlib.pyplot as plt # noqa + + # Get auto data + autos = {} + amps = {} + times = {} + if args.log: + autos_raw = {} + if len(args.files) == 0: + # No file given, use redis db + redis = redis_lib.Redis('redishost') + keys = [k for k in redis.keys() if k.startswith('visdata')] + for key in keys: + ant = int(re.findall(r'visdata://(\d+)/', key)[0]) + pol = key[-2:] + autos[(ant, pol)] = np.fromstring(redis.hgetall(key).get('data'), dtype=np.float32) + amps[(ant, pol)] = np.median(autos[(ant, pol)]) + if args.log: + autos_raw[(ant, pol)] = autos[(ant, pol)] + autos[(ant, pol)] = 10.0 * np.log10(autos[(ant, pol)]) + amps[(ant, pol)] = 10.0 * np.log10(amps[(ant, pol)]) + times[(ant, pol)] = float(redis.hgetall(key).get('time', 0)) + else: + counts = {} + for fname in args.files: + uvd = apm.UV(fname) + pol = uvutils.polnum2str(uvd['pol']).lower() + uvd.select('auto', 1, 1) + for (uvw, this_t, (ant1, ant2)), auto, fname in uvd.all(raw=True): + try: + counts[(ant1, pol)] += 1 + autos[(ant1, pol)] += auto + times[(ant1, pol)] += this_t + except KeyError: + counts[(ant1, pol)] = 1 + autos[(ant1, pol)] = auto + times[(ant1, pol)] = this_t + for key in autos.keys(): + autos[key] /= counts[key] + times[key] /= counts[key] + amps[key] = np.median(autos[key]) + if args.log: + autos_raw[key] = autos[key] + autos[key] = 10.0 * np.log10(autos[key]) + amps[key] = 10.0 * np.log10(amps[key]) + del(uvd) + del(counts) + + ants = np.unique([ant for (ant, pol) in autos.keys()]) + # Find most recent time, only keep spectra from that time + latest = np.max(times.values()) + for key, this_t in times.items(): + if latest - this_t > 1. / 60. / 24.: + # more than a minute from latest, use NaN to flag + autos[key] = np.nan + amps[key] = np.nan + latest = time.Time(latest, format='jd') + + # Get cminfo + parser = mc.get_mc_argument_parser() + mcargs = parser.parse_args(args=[]) # args=[] to throw away command line arguments + db = mc.connect_to_mc_db(mcargs) + session = db.sessionmaker() + hsession = sys_handling.Handling(session) + stations_conn = hsession.get_all_fully_connected_at_date(at_date=latest) + antpos = np.zeros((np.max(ants) + 1, 2)) + ants_connected = [] + antnames = ["" for x in range(np.max(ants) + 1)] + for stn in stations_conn: + # make sure that antenna in cminfo is in data file + try: + antpos[stn['antenna_number'], :] = [stn['easting'], stn['northing']] + except IndexError: + continue + ants_connected.append(stn['antenna_number']) + antnames[stn['antenna_number']] = stn['station_name'] + array_center = np.mean(antpos[antpos[:, 0] != 0, :], axis=0) + antpos -= array_center + + # Get receiverator and PAM info + receiverators = ["" for x in range(np.max(ants) + 1)] + rxr_nums = np.zeros(np.max(ants) + 1, dtype=int) + pams = ["" for x in range(np.max(ants) + 1)] + for ant in ants_connected: + pinfo = hsession.get_pam_info(antnames[ant], latest) + receiverators[ant] = pinfo['e'][0][:-1] + pams[ant] = pinfo['e'][1] + result = re.findall(r'RI(\d+)', pinfo['e'][0])[0] + rxr_nums[ant] = int(result) + + # Pick a colormap to highlight "good", "bad", and "really bad" + # TODO: Fine tune this + goodbad = plt.get_cmap('RdYlGn') + + # Construct path stuff for output + if args.outpath == '': + # default path is same directory as file + try: + outpath = os.path.dirname(os.path.abspath(args.files[0])) + except IndexError: + outpath = os.path.abspath(os.path.curdir) + else: + outpath = args.outpath + if args.outbase == '': + try: + basename = '.'.join(os.path.basename(args.files[0]).split('.')[0:3]) + except IndexError: + basename = '%5f' % latest.jd + else: + basename = args.outbase + + # Plot autos vs positions + pol_labels = {'xx': 'E', 'yy': 'N'} + poli = {'xx': 0, 'yy': 1} + if args.log: + vmin = -30 + vmax = 15 + else: + vmin = 0 + vmax = 12 + fig = plt.figure(figsize=(10, 8)) + for ant in ants_connected: + for pol in ['xx', 'yy']: + try: + if not np.isnan(amps[(ant, pol)]): + ax = plt.scatter(antpos[ant, 0], antpos[ant, 1] + 3 * (poli[pol] - 0.5), + c=amps[(ant, pol)], vmin=vmin, vmax=vmax, cmap=goodbad) + else: + plt.scatter(antpos[ant, 0], antpos[ant, 1] + 3 * (poli[pol] - 0.5), + marker='x', color='k') + except KeyError: + plt.scatter(antpos[ant, 0], antpos[ant, 1] + 3 * (poli[pol] - 0.5), + marker='x', color='k') + text = (str(ant) + '\n' + pams[ant] + '\n' + receiverators[ant]) + plt.annotate(text, xy=antpos[ant, 0:2] + [1, 0], textcoords='data', + verticalalignment='center') + if args.log: + label = '10log10(Median Autos)' + else: + label = 'Median Autos' + plt.colorbar(ax, label=label) + xr = antpos[ants_connected, 0].max() - antpos[ants_connected, 0].min() + yr = antpos[ants_connected, 1].max() - antpos[ants_connected, 1].min() + plt.xlim([antpos[ants_connected, 0].min() - 0.05 * xr, antpos[ants_connected, 0].max() + 0.2 * xr]) + plt.ylim([antpos[ants_connected, 1].min() - 0.05 * yr, antpos[ants_connected, 1].max() + 0.1 * yr]) + plt.title(str(latest.datetime) + ' UTC; JD=' + str(latest.jd)) + # Add polarization key + for pol in ['xx', 'yy']: + xpos = antpos[ants_connected, 0].min() + ypos = antpos[ants_connected, 1].min() + 3 * (poli[pol]) + plt.scatter(xpos, ypos, c=vmax, vmin=vmin, vmax=vmax, cmap=goodbad) + plt.annotate(pol_labels[pol] + ' pol', xy=[xpos + 1, ypos], textcoords='data', verticalalignment='center') + xmin = plt.gca().get_xlim()[0] + ymin = plt.gca().get_ylim()[0] + plt.plot([xmin, xmin + xr / 6., xmin + xr / 6.], [ymin * 3. / 4., ymin * 3. / 4., ymin], 'k') + # Save file + filename = os.path.join(outpath, basename + '.auto_v_pos.png') + plt.savefig(filename) + + + # Autos v rxr + fig, axarr = plt.subplots(1, np.max(rxr_nums), sharex=True, sharey=True, figsize=(15, 8)) + for rxr in np.unique(rxr_nums[ants_connected]): + ind = np.where(rxr_nums == rxr)[0] + for anti, ant in enumerate(ind): + for pol in ['xx', 'yy']: + try: + if not np.isnan(amps[(ant, pol)]): + ax = axarr[rxr - 1].scatter(0, anti + 0.3 * poli[pol], c=amps[(ant, pol)], + vmin=vmin, vmax=vmax, cmap=goodbad) + else: + axarr[rxr - 1].scatter(0, anti + 0.3 * poli[pol], marker='x', color='k') + except BaseException: + axarr[rxr - 1].scatter(0, anti + 0.3 * poli[pol], marker='x', color='k') + axarr[rxr - 1].annotate(str(ant) + ',' + pams[ant], xy=[0.01, i]) + axarr[rxr - 1].set_yticks([]) + axarr[rxr - 1].set_xticks([]) + for rxr in range(np.max(rxr_nums)): + axarr[rxr].set_title('Rxr ' + str(rxr + 1)) + plt.xlim([-.01, .1]) + plt.subplots_adjust(bottom=0.15) + plt.subplots_adjust(wspace=0) + # cbar_ax = f.add_axes([.14, .05, .72, .05]) + cbar_ax = fig.add_axes([.13, .05, .67, .05]) + if args.log: + label = '10log10(Median Autos)' + else: + label = 'Median Autos' + fig.colorbar(ax, cax=cbar_ax, orientation='horizontal', label=label) + fig.suptitle(str(latest.datetime) + ' UTC') + # Add polarization key + p_ax = fig.add_axes([0.82, 0.03, 0.06, 0.08]) + for pol in ['xx', 'yy']: + yval = poli[pol] + plt.scatter(0, yval, c=vmax, vmin=vmin, vmax=vmax, cmap=goodbad) + plt.annotate(pol_labels[pol] + ' pol', xy=[1, yval], textcoords='data', verticalalignment='center') + plt.xlim([-1, 4]) + plt.ylim([-1, 2]) + p_ax.set_xticks([]) + p_ax.set_yticks([]) + # Save file + filename = os.path.join(outpath, basename + '.auto_v_rxr.png') + plt.savefig(filename) + + # Plot spectra + fig = plt.figure(figsize=(20, 12)) + # Number of subplots in X and Y directions + nants = len(ants) + nx = int(np.ceil(np.log2(nants + 1))) + ny = int(np.ceil(nants / float(nx))) + pol_colors = {'xx': 'r', 'yy': 'b'} + ants = np.sort(ants) + for ai, ant in enumerate(ants): + ax = plt.subplot(nx, ny, ai + 1) + for pol in ['xx', 'yy']: + try: + plt.plot(autos[(ant, pol)], pol_colors[pol], label=pol_labels[pol]) + except KeyError: + continue + if ant in ants_connected: + lcolor = 'r' + else: + lcolor = 'k' + plt.text(0.8, 0.8, str(ant), fontsize=12, transform=ax.transAxes, color=lcolor) + # Axis stuff + ax.axes.get_xaxis().set_ticklabels([]) + if ai % ny: + ax.axes.get_yaxis().set_ticklabels([]) + else: + if args.log: + ax.set_ylabel('10log10') + else: + ax.set_ylabel('linear') + ax.set_ylim([vmin, 1.3 * vmax]) + if ai == 0: + plt.legend(loc='best') + fig.suptitle(str(latest.datetime) + ' UTC; JD=' + str(latest.jd)) + filename = os.path.join(outpath, basename + '.auto_specs.png') + plt.savefig(filename) + + # Plot rms values at input of ADC's + + + def plot_rms(data, POL='xx'): + """Plots rms values at input to ADC's by looking at the autos. + + Args: + data (dict): Dictionary of auto's. + POL (str): String of polarization.""" + fig = plt.figure(figsize=(20, 12)) + CHUNK = 256 + BINS = 24 + rmscolors = 'bgrcmy' + ants = sorted([i for (i, pp) in data if pp == POL]) + Nrow = np.ceil(np.sqrt(len(ants))) + Mcol = np.ceil(len(ants) / float(Nrow)) + bins = np.logspace(-2, 4, BINS, base=2.) + for cnt, ant in enumerate(ants): + ax = plt.subplot(Nrow, Mcol, cnt + 1) + for ch in xrange(0, 1024, CHUNK): + dchunk = data[ant, POL][ch:ch + CHUNK].flatten() + hist, hbins = np.histogram(np.sqrt(dchunk / 2), bins) + hist = 10**np.log10(hist + .1) + hbins = 0.5 * (hbins[1:] + hbins[:-1]) + ax.fill_between(np.log2(bins), hist, .1, where=hist > .1, color=rmscolors[j], alpha=.5) + bounds = np.where(bins < 2**0, dchunk.size, np.where(bins > 2**2, dchunk.size, 0)) + ax.fill_between(np.log2(bins), bounds, .1, where=bounds > .1, color='black', alpha=.6) + ax.set_yscale('log') + plt.xlim(-2, 3) + plt.ylim(d.size / 1e2, d.size) + if ant in ants_connected: + lcolor = 'r' + else: + lcolor = 'b' + plt.text(0.5, 0.8, str(ant), fontsize=12, transform=ax.transAxes, color=lcolor) + ax.get_yaxis().set_visible(False) + if cnt < (Nrow - 1) * Mcol: + ax.get_xaxis().set_ticklabels([]) + else: + plt.xlabel(r'$V_{\rm rms}$ [bits]') + plt.grid() + fig.suptitle(str(latest.datetime) + ' UTC' + ' {0}'.format(POL) + '; JD=' + str(latest.jd)) + plt.tight_layout(rect=(0, 0, 1, .95)) + + + for pol in ['xx', 'yy']: + if args.log: + plot_rms(autos_raw, POL=pol) + else: + plot_rms(autos, POL=pol) + filename = os.path.join(outpath, basename + '.{0}.auto_rms_values.png'.format(pol)) + plt.savefig(filename) + + # ID some potential baddies + if args.idbaddies: + baddies = [str((key[0], pol_labels[key[1]])) for key, val in amps.items() if + val < 0.75 * (vmax - vmin) + vmin and key[0] in ants_connected] + filename = os.path.join(outpath, basename + '.baddies.txt') + np.savetxt(filename, baddies, fmt='%s', header='You may want to check these antennas:') + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/hera_qm/scripts/delay_xrfi_h1c_idr2_1_run.py b/hera_qm/scripts/delay_xrfi_h1c_idr2_1_run.py new file mode 100755 index 00000000..66c7edea --- /dev/null +++ b/hera_qm/scripts/delay_xrfi_h1c_idr2_1_run.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +import sys +import numpy as np +from hera_qm import utils as qm_utils +from hera_qm import xrfi +from hera_cal import delay_filter +from hera_cal import io +from pyuvdata import UVData + +def main(): + ap = qm_utils.get_metrics_ArgumentParser('delay_xrfi_h1c_idr2_1_run') + args = ap.parse_args() + filename = args.filename + history = ' '.join(sys.argv) + + # Read data, apply delay filter, update UVData object + uv = UVData() + uv.read_miriad(filename) + # apply a priori waterfall flags + if args.waterfalls is not None: + waterfalls = args.waterfalls.split(',') + if len(waterfalls) > 0: + xrfi.flag_apply(waterfalls, uv, force_pol=True) + + # set kwargs + kwargs = {} + if args.window == 'tukey': + kwargs['alpha'] = args.alpha + + # Stuff into delay filter object, run delay filter + dfil = delay_filter.Delay_Filter() + dfil.load_data(uv) + dfil.run_filter(standoff=args.standoff, horizon=args.horizon, tol=args.tol, + window=args.window, skip_wgt=args.skip_wgt, maxiter=args.maxiter, **kwargs) + io.update_uvdata(dfil.input_data, data=dfil.filtered_residuals, flags=dfil.flags) + + # Run xrfi + xrfi.xrfi_h1c_run(dfil.input_data, history, infile_format=args.infile_format, + extension=args.extension, summary=args.summary, summary_ext=args.summary_ext, + xrfi_path=args.xrfi_path, model_file=args.model_file, + model_file_format=args.model_file_format, calfits_file=args.calfits_file, + kt_size=args.kt_size, kf_size=args.kf_size, sig_init=args.sig_init, + sig_adj=args.sig_adj, px_threshold=args.px_threshold, + freq_threshold=args.freq_threshold, time_threshold=args.time_threshold, + ex_ants=args.ex_ants, metrics_file=args.metrics_file, filename=args.filename) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/firstcal_metrics_example.ipynb b/hera_qm/scripts/firstcal_metrics_example.ipynb similarity index 100% rename from scripts/firstcal_metrics_example.ipynb rename to hera_qm/scripts/firstcal_metrics_example.ipynb diff --git a/hera_qm/scripts/firstcal_metrics_run.py b/hera_qm/scripts/firstcal_metrics_run.py new file mode 100755 index 00000000..b0673b62 --- /dev/null +++ b/hera_qm/scripts/firstcal_metrics_run.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +from hera_qm import utils +from hera_qm import firstcal_metrics +import sys + +def main(): + ap = utils.get_metrics_ArgumentParser('firstcal_metrics') + args = ap.parse_args() + files = args.files + history = ' '.join(sys.argv) + + firstcal_metrics.firstcal_metrics_run(files, args, history) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/omnical_metrics_example.ipynb b/hera_qm/scripts/omnical_metrics_example.ipynb similarity index 100% rename from scripts/omnical_metrics_example.ipynb rename to hera_qm/scripts/omnical_metrics_example.ipynb diff --git a/hera_qm/scripts/omnical_metrics_run.py b/hera_qm/scripts/omnical_metrics_run.py new file mode 100755 index 00000000..3d99b1eb --- /dev/null +++ b/hera_qm/scripts/omnical_metrics_run.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +from hera_qm import utils +from hera_qm import omnical_metrics +import sys + +def main(): + ap = utils.get_metrics_ArgumentParser('omnical_metrics') + args = ap.parse_args() + files = args.files + history = ' '.join(sys.argv) + + omnical_metrics.omnical_metrics_run(files, args, history) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/test_coverage.sh b/hera_qm/scripts/test_coverage.sh similarity index 100% rename from scripts/test_coverage.sh rename to hera_qm/scripts/test_coverage.sh diff --git a/scripts/test_red_corr_metrics.ipynb b/hera_qm/scripts/test_red_corr_metrics.ipynb similarity index 100% rename from scripts/test_red_corr_metrics.ipynb rename to hera_qm/scripts/test_red_corr_metrics.ipynb diff --git a/scripts/test_xrfi_performance.py b/hera_qm/scripts/test_xrfi_performance.py similarity index 100% rename from scripts/test_xrfi_performance.py rename to hera_qm/scripts/test_xrfi_performance.py diff --git a/hera_qm/scripts/xrfi_apply.py b/hera_qm/scripts/xrfi_apply.py new file mode 100755 index 00000000..16f1489e --- /dev/null +++ b/hera_qm/scripts/xrfi_apply.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +import sys +from hera_qm import utils +from hera_qm import xrfi + +def main(): + ap = utils.get_metrics_ArgumentParser('xrfi_apply') + args = ap.parse_args() + filename = args.filename + history = ' '.join(sys.argv) + + xrfi.xrfi_h1c_apply(filename, history, infile_format=args.infile_format, xrfi_path=args.xrfi_path, + outfile_format=args.outfile_format, extension=args.extension, + overwrite=args.overwrite, flag_file=args.flag_file, waterfalls=args.waterfalls, + output_uvflag=args.output_uvflag, output_uvflag_ext=args.output_uvflag_ext) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/hera_qm/scripts/xrfi_day_threshold_run.py b/hera_qm/scripts/xrfi_day_threshold_run.py new file mode 100755 index 00000000..eca25276 --- /dev/null +++ b/hera_qm/scripts/xrfi_day_threshold_run.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +import sys +from hera_qm import utils +from hera_qm import xrfi + +def main(): + ap = utils.get_metrics_ArgumentParser('day_threshold_run') + args = ap.parse_args() + history = ' '.join(sys.argv) + + if args.run_if_first is None or sorted(args.data_files)[0] == args.run_if_first: + xrfi.day_threshold_run(args.data_files, history, nsig_f=args.nsig_f, nsig_t=args.nsig_t, flag_abscal=not(args.skip_making_flagged_abs_calfits), + nsig_f_adj=args.nsig_f_adj, nsig_t_adj=args.nsig_t_adj, clobber=args.clobber, + a_priori_flag_yaml=args.a_priori_flag_yaml) + else: + print(sorted(args.data_files)[0], 'is not', args.run_if_first, '...skipping.') + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/hera_qm/scripts/xrfi_h1c_run.py b/hera_qm/scripts/xrfi_h1c_run.py new file mode 100755 index 00000000..962785dd --- /dev/null +++ b/hera_qm/scripts/xrfi_h1c_run.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +import sys +from hera_qm import utils +from hera_qm import xrfi + +def main(): + ap = utils.get_metrics_ArgumentParser('xrfi_h1c_run') + args = ap.parse_args() + filename = args.filename + history = ' '.join(sys.argv) + + xrfi.xrfi_h1c_run(filename, history, infile_format=args.infile_format, + extension=args.extension, summary=args.summary, summary_ext=args.summary_ext, + xrfi_path=args.xrfi_path, model_file=args.model_file, + model_file_format=args.model_file_format, calfits_file=args.calfits_file, + kt_size=args.kt_size, kf_size=args.kf_size, sig_init=args.sig_init, + sig_adj=args.sig_adj, px_threshold=args.px_threshold, + freq_threshold=args.freq_threshold, time_threshold=args.time_threshold, + ex_ants=args.ex_ants, metrics_file=args.metrics_file, filename=filename[0]) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/hera_qm/scripts/xrfi_h3c_idr2_1_run.py b/hera_qm/scripts/xrfi_h3c_idr2_1_run.py new file mode 100755 index 00000000..d6a8eb07 --- /dev/null +++ b/hera_qm/scripts/xrfi_h3c_idr2_1_run.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2020 the HERA Project +# Licensed under the MIT License + +import sys +from hera_qm import utils +from hera_qm import xrfi + +def main(): + ap = utils.get_metrics_ArgumentParser('xrfi_h3c_idr2_1_run') + args = ap.parse_args() + flag_command = ' '.join(sys.argv) + + xrfi.xrfi_h3c_idr2_1_run(args.ocalfits_files, args.acalfits_files, args.model_files, + args.data_files, flag_command, xrfi_path=args.xrfi_path, + kt_size=args.kt_size, kf_size=args.kf_size, sig_init=args.sig_init, + sig_adj=args.sig_adj, ex_ants=args.ex_ants, + metrics_file=args.metrics_file, clobber=args.clobber) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/hera_qm/scripts/xrfi_h3ca_rtp_run.py b/hera_qm/scripts/xrfi_h3ca_rtp_run.py new file mode 100755 index 00000000..7429c71d --- /dev/null +++ b/hera_qm/scripts/xrfi_h3ca_rtp_run.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +import sys +from hera_qm import utils +from hera_qm import xrfi + +def main(): + ap = utils.get_metrics_ArgumentParser('xrfi_run') # Use same parser as IDR2.2 script + args = ap.parse_args() + history = ' '.join(sys.argv) + + xrfi.xrfi_run(args.data_file, history, xrfi_path=args.xrfi_path, + kt_size=args.kt_size, kf_size=args.kf_size, sig_init=args.sig_init, + sig_adj=args.sig_adj, ex_ants=args.ex_ants, + metrics_file=args.metrics_file, clobber=args.clobber) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/xrfi_revamp_2018.ipynb b/hera_qm/scripts/xrfi_revamp_2018.ipynb similarity index 100% rename from scripts/xrfi_revamp_2018.ipynb rename to hera_qm/scripts/xrfi_revamp_2018.ipynb diff --git a/hera_qm/scripts/xrfi_run.py b/hera_qm/scripts/xrfi_run.py new file mode 100755 index 00000000..c1278396 --- /dev/null +++ b/hera_qm/scripts/xrfi_run.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +import sys +from hera_qm import utils +from hera_qm import xrfi + +def main(): + ap = utils.get_metrics_ArgumentParser('xrfi_run') + args = ap.parse_args() + history = ' '.join(sys.argv) + + xrfi.xrfi_run(args.ocalfits_files, + args.acalfits_files, + args.model_files, + args.data_files, + a_priori_flag_yaml=args.a_priori_flag_yaml, + a_priori_ants_only=not(args.a_apriori_times_and_freqs), + use_cross_pol_vis=not(args.skip_cross_pol_vis), + history=history, + xrfi_path=args.xrfi_path, + throw_away_edges=not(args.keep_edge_times), + kt_size=args.kt_size, + kf_size=args.kf_size, + sig_init_med=args.sig_init_med, + sig_adj_med=args.sig_adj_med, + sig_init_mean=args.sig_init_mean, + sig_adj_mean=args.sig_adj_mean, + ex_ants=args.ex_ants, + Nwf_per_load=args.Nwf_per_load, + omnical_median_filter=not(args.skip_omnical_median_filter), + omnical_mean_filter=not(args.skip_omnical_mean_filter), + omnical_chi2_median_filter=not(args.skip_omnical_chi2_median_filter), + omnical_chi2_mean_filter=not(args.skip_omnical_chi2_mean_filter), + omnical_zscore_filter=not(args.skip_omnical_zscore_filter), + abscal_median_filter=not(args.skip_abscal_median_filter), + abscal_mean_filter=not(args.skip_abscal_mean_filter), + abscal_chi2_median_filter=not(args.skip_abscal_chi2_median_filter), + abscal_chi2_mean_filter=not(args.skip_abscal_chi2_mean_filter), + abscal_zscore_filter=not(args.skip_abscal_zscore_filter), + omnivis_median_filter=not(args.skip_omnivis_median_filter), + omnivis_mean_filter=not(args.skip_omnivis_mean_filter), + auto_median_filter=not(args.skip_auto_median_filter), + auto_mean_filter=not(args.skip_auto_mean_filter), + cross_median_filter=args.use_cross_median_filter, + cross_mean_filter=not(args.skip_cross_mean_filter), + metrics_files=args.metrics_files, clobber=args.clobber) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/hera_qm/scripts/xrfi_run_data_only.py b/hera_qm/scripts/xrfi_run_data_only.py new file mode 100644 index 00000000..6b2708a9 --- /dev/null +++ b/hera_qm/scripts/xrfi_run_data_only.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2019 the HERA Project +# Licensed under the MIT License + +import sys +from hera_qm import utils +from hera_qm import xrfi + +def main(): + ap = utils.get_metrics_ArgumentParser('xrfi_run_data_only') + args = ap.parse_args() + history = ' '.join(sys.argv) + + xrfi.xrfi_run(data_files=args.data_files, + a_priori_flag_yaml=args.a_priori_flag_yaml, + use_cross_pol_vis=not(args.skip_cross_pol_vis), + cross_median_filter=args.cross_median_filter, + cross_mean_filter=not(args.skip_cross_mean_filter), + history=history, + xrfi_path=args.xrfi_path, + throw_away_edges=not(args.keep_edge_times), + kt_size=args.kt_size, + kf_size=args.kf_size, + sig_init_med=args.sig_init_med, + sig_adj_med=args.sig_adj_med, + sig_init_mean=args.sig_init_mean, + sig_adj_mean=args.sig_adj_mean, + ex_ants=args.ex_ants, + Nwf_per_load=args.Nwf_per_load, + metrics_files=args.metrics_files, + clobber=args.clobber) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b0670f7f..c033f28c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,77 @@ [build-system] -requires = ["setuptools>=30.3.0", "wheel", "setuptools_scm[toml]>=6.2"] +requires = ["setuptools>=60", "wheel", "setuptools_scm>=8.0"] build-backend = "setuptools.build_meta" +[project] +name="hera_qm" +authors=[ + {name = "HERA Team"} +] +description="HERA Data Quality Metrics." +license= {text= "MIT"} +requires-python = ">=3.9" +dependencies = [ + "numpy>1.23", + "astropy>=5.0.4", + "h5py>=3.1", + "pyyaml", + "pyuvdata>=2.3", + "hera_filters" +] +classifiers = [ + "Programming Language :: Python :: 3", +] +dynamic = ["readme", "version"] + +[project.urls] +repository="https://github.com/HERA-Team/hera_qm" +# documentation="https://hera_qm.readthedocs.io" + +[tool.setuptools.dynamic] +readme = {file = ["README.md"]} + +[project.optional-dependencies] +tests = [ + "pytest", + "pytest-cov", +] +all = [ + "hera_qm[tests]" +] + +[project.scripts] +"ant_metrics_run.py" = "hera_qm.scripts.ant_metrics_run:main" +"auto_metrics_run.py" = "hera_qm.scripts.auto_metrics_run:main" +"xrfi_run.py" = "hera_qm.scripts.xrfi_run:main" +"firstcal_metrics_run.py" = "hera_qm.scripts.firstcal_metrics_run:main" +"auto_view.py" = "hera_qm.scripts.auto_view:main" +"omnical_metrics_run.py" = "hera_qm.scripts.omnical_metrics_run:main" +"xrfi_apply.py" = "hera_qm.scripts.xrfi_apply:main" +"delay_xrfi_h1c_idr2_1_run.py" = "hera_qm.scripts.delay_xrfi_h1c_idr2_1_run:main" +"xrfi_h1c_run.py" = "hera_qm.scripts.xrfi_h1c_run:main" +"xrfi_day_threshold_run.py" = "hera_qm.scripts.xrfi_day_threshold_run:main" +"xrfi_h3c_idr2_1_run.py" = "hera_qm.scripts.xrfi_h3c_idr2_1_run:main" +"xrfi_h3ca_rtp_run.py" = "hera_qm.scripts.xrfi_h3ca_rtp_run:main" +"xrfi_run_data_only.py" = "hera_qm.scripts.xrfi_run_data_only:main" + [tool.setuptools_scm] write_to = "hera_qm/_version.py" parentdir_prefix_version = "hera_qm-" fallback_version = "0.0.0" + + +[tool.pytest.ini_options] +addopts = "--ignore=hera_qm/scripts --cov=hera_qm --cov-config=.coveragerc" + +[tool.ruff] +line-length = 88 +target-version = "py39" + +[tool.ruff.lint] +ignore = [ + "E501", + "W503" +] + +[tool.ruff.lint.pydocstyle] +convention = "numpy" diff --git a/scripts/ant_metrics_run.py b/scripts/ant_metrics_run.py deleted file mode 100755 index d247e810..00000000 --- a/scripts/ant_metrics_run.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -from hera_qm import utils -from hera_qm import ant_metrics -import sys - -ap = utils.get_metrics_ArgumentParser('ant_metrics') -args = ap.parse_args() -history = ' '.join(sys.argv) -ant_metrics.ant_metrics_run(args.sum_files, - diff_files=args.diff_files, - apriori_xants=args.apriori_xants, - a_priori_xants_yaml=args.a_priori_xants_yaml, - crossCut=args.crossCut, - deadCut=args.deadCut, - metrics_path=args.metrics_path, - extension=args.extension, - overwrite=args.clobber, - verbose=args.verbose, - history=history) diff --git a/scripts/auto_metrics_run.py b/scripts/auto_metrics_run.py deleted file mode 100755 index 1a550dfc..00000000 --- a/scripts/auto_metrics_run.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2021 the HERA Project -# Licensed under the MIT License - -from hera_qm import utils -from hera_qm.auto_metrics import auto_metrics_run -import sys - -ap = utils.get_metrics_ArgumentParser('auto_metrics') -args = ap.parse_args() -history = ' '.join(sys.argv) - -auto_metrics_run(args.metric_outfile, - args.raw_auto_files, - median_round_modz_cut=args.median_round_modz_cut, - mean_round_modz_cut=args.mean_round_modz_cut, - edge_cut=args.edge_cut, - Kt=args.Kt, - Kf=args.Kf, - sig_init=args.sig_init, - sig_adj=args.sig_adj, - chan_thresh_frac=args.chan_thresh_frac, - history=history, - overwrite=args.clobber) diff --git a/scripts/auto_view.py b/scripts/auto_view.py deleted file mode 100755 index be7d39d7..00000000 --- a/scripts/auto_view.py +++ /dev/null @@ -1,337 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -import os -import argparse -import numpy as np -import re -import matplotlib as mpl -import redis as redis_lib -from hera_mc import mc, sys_handling, cm_utils -from astropy import time -from matplotlib.colors import LogNorm -from matplotlib.colors import LinearSegmentedColormap -import pyuvdata.utils as uvutils -import aipy.miriad as apm - - -parser = argparse.ArgumentParser(description='Plot auto locations and magnitudes') -parser.add_argument('-s', '--show', dest='show', action='store_true', default=False, - help='Show the plot. Default: False -- i.e., just save a png') -parser.add_argument('-l', '--log', dest='log', action='store_true', default=True, - help='Take 10*log10() of data before plotting. Default:True') -parser.add_argument('--outpath', default='', type=str, - help='Path to save output plots to. Default is same directory as file.') -parser.add_argument('--outbase', default='', type=str, - help='Base for output file names. Default is JD of data.') -parser.add_argument('--idbaddies', dest='idbaddies', action='store_true', default=False, - help='Identify potential misbehaving antennas. Default: False') -parser.add_argument('files', metavar='files', type=str, nargs='*', default=[], - help='Files for which to plot auto views.') -args = parser.parse_args() - -if not args.show: - # use a matplotlib backend that doesn't require an X session - mpl.use('Agg') -# this has to be called after matplotlibs use() -import matplotlib.pyplot as plt # noqa - -# Get auto data -autos = {} -amps = {} -times = {} -if args.log: - autos_raw = {} -if len(args.files) == 0: - # No file given, use redis db - redis = redis_lib.Redis('redishost') - keys = [k for k in redis.keys() if k.startswith('visdata')] - for key in keys: - ant = int(re.findall(r'visdata://(\d+)/', key)[0]) - pol = key[-2:] - autos[(ant, pol)] = np.fromstring(redis.hgetall(key).get('data'), dtype=np.float32) - amps[(ant, pol)] = np.median(autos[(ant, pol)]) - if args.log: - autos_raw[(ant, pol)] = autos[(ant, pol)] - autos[(ant, pol)] = 10.0 * np.log10(autos[(ant, pol)]) - amps[(ant, pol)] = 10.0 * np.log10(amps[(ant, pol)]) - times[(ant, pol)] = float(redis.hgetall(key).get('time', 0)) -else: - counts = {} - for fname in args.files: - uvd = apm.UV(fname) - pol = uvutils.polnum2str(uvd['pol']).lower() - uvd.select('auto', 1, 1) - for (uvw, this_t, (ant1, ant2)), auto, fname in uvd.all(raw=True): - try: - counts[(ant1, pol)] += 1 - autos[(ant1, pol)] += auto - times[(ant1, pol)] += this_t - except KeyError: - counts[(ant1, pol)] = 1 - autos[(ant1, pol)] = auto - times[(ant1, pol)] = this_t - for key in autos.keys(): - autos[key] /= counts[key] - times[key] /= counts[key] - amps[key] = np.median(autos[key]) - if args.log: - autos_raw[key] = autos[key] - autos[key] = 10.0 * np.log10(autos[key]) - amps[key] = 10.0 * np.log10(amps[key]) - del(uvd) - del(counts) - -ants = np.unique([ant for (ant, pol) in autos.keys()]) -# Find most recent time, only keep spectra from that time -latest = np.max(times.values()) -for key, this_t in times.items(): - if latest - this_t > 1. / 60. / 24.: - # more than a minute from latest, use NaN to flag - autos[key] = np.nan - amps[key] = np.nan -latest = time.Time(latest, format='jd') - -# Get cminfo -parser = mc.get_mc_argument_parser() -mcargs = parser.parse_args(args=[]) # args=[] to throw away command line arguments -db = mc.connect_to_mc_db(mcargs) -session = db.sessionmaker() -hsession = sys_handling.Handling(session) -stations_conn = hsession.get_all_fully_connected_at_date(at_date=latest) -antpos = np.zeros((np.max(ants) + 1, 2)) -ants_connected = [] -antnames = ["" for x in range(np.max(ants) + 1)] -for stn in stations_conn: - # make sure that antenna in cminfo is in data file - try: - antpos[stn['antenna_number'], :] = [stn['easting'], stn['northing']] - except IndexError: - continue - ants_connected.append(stn['antenna_number']) - antnames[stn['antenna_number']] = stn['station_name'] -array_center = np.mean(antpos[antpos[:, 0] != 0, :], axis=0) -antpos -= array_center - -# Get receiverator and PAM info -receiverators = ["" for x in range(np.max(ants) + 1)] -rxr_nums = np.zeros(np.max(ants) + 1, dtype=int) -pams = ["" for x in range(np.max(ants) + 1)] -for ant in ants_connected: - pinfo = hsession.get_pam_info(antnames[ant], latest) - receiverators[ant] = pinfo['e'][0][:-1] - pams[ant] = pinfo['e'][1] - result = re.findall(r'RI(\d+)', pinfo['e'][0])[0] - rxr_nums[ant] = int(result) - -# Pick a colormap to highlight "good", "bad", and "really bad" -# TODO: Fine tune this -goodbad = plt.get_cmap('RdYlGn') - -# Construct path stuff for output -if args.outpath == '': - # default path is same directory as file - try: - outpath = os.path.dirname(os.path.abspath(args.files[0])) - except IndexError: - outpath = os.path.abspath(os.path.curdir) -else: - outpath = args.outpath -if args.outbase == '': - try: - basename = '.'.join(os.path.basename(args.files[0]).split('.')[0:3]) - except IndexError: - basename = '%5f' % latest.jd -else: - basename = args.outbase - -# Plot autos vs positions -pol_labels = {'xx': 'E', 'yy': 'N'} -poli = {'xx': 0, 'yy': 1} -if args.log: - vmin = -30 - vmax = 15 -else: - vmin = 0 - vmax = 12 -fig = plt.figure(figsize=(10, 8)) -for ant in ants_connected: - for pol in ['xx', 'yy']: - try: - if not np.isnan(amps[(ant, pol)]): - ax = plt.scatter(antpos[ant, 0], antpos[ant, 1] + 3 * (poli[pol] - 0.5), - c=amps[(ant, pol)], vmin=vmin, vmax=vmax, cmap=goodbad) - else: - plt.scatter(antpos[ant, 0], antpos[ant, 1] + 3 * (poli[pol] - 0.5), - marker='x', color='k') - except KeyError: - plt.scatter(antpos[ant, 0], antpos[ant, 1] + 3 * (poli[pol] - 0.5), - marker='x', color='k') - text = (str(ant) + '\n' + pams[ant] + '\n' + receiverators[ant]) - plt.annotate(text, xy=antpos[ant, 0:2] + [1, 0], textcoords='data', - verticalalignment='center') -if args.log: - label = '10log10(Median Autos)' -else: - label = 'Median Autos' -plt.colorbar(ax, label=label) -xr = antpos[ants_connected, 0].max() - antpos[ants_connected, 0].min() -yr = antpos[ants_connected, 1].max() - antpos[ants_connected, 1].min() -plt.xlim([antpos[ants_connected, 0].min() - 0.05 * xr, antpos[ants_connected, 0].max() + 0.2 * xr]) -plt.ylim([antpos[ants_connected, 1].min() - 0.05 * yr, antpos[ants_connected, 1].max() + 0.1 * yr]) -plt.title(str(latest.datetime) + ' UTC; JD=' + str(latest.jd)) -# Add polarization key -for pol in ['xx', 'yy']: - xpos = antpos[ants_connected, 0].min() - ypos = antpos[ants_connected, 1].min() + 3 * (poli[pol]) - plt.scatter(xpos, ypos, c=vmax, vmin=vmin, vmax=vmax, cmap=goodbad) - plt.annotate(pol_labels[pol] + ' pol', xy=[xpos + 1, ypos], textcoords='data', verticalalignment='center') -xmin = plt.gca().get_xlim()[0] -ymin = plt.gca().get_ylim()[0] -plt.plot([xmin, xmin + xr / 6., xmin + xr / 6.], [ymin * 3. / 4., ymin * 3. / 4., ymin], 'k') -# Save file -filename = os.path.join(outpath, basename + '.auto_v_pos.png') -plt.savefig(filename) - - -# Autos v rxr -fig, axarr = plt.subplots(1, np.max(rxr_nums), sharex=True, sharey=True, figsize=(15, 8)) -for rxr in np.unique(rxr_nums[ants_connected]): - ind = np.where(rxr_nums == rxr)[0] - for anti, ant in enumerate(ind): - for pol in ['xx', 'yy']: - try: - if not np.isnan(amps[(ant, pol)]): - ax = axarr[rxr - 1].scatter(0, anti + 0.3 * poli[pol], c=amps[(ant, pol)], - vmin=vmin, vmax=vmax, cmap=goodbad) - else: - axarr[rxr - 1].scatter(0, anti + 0.3 * poli[pol], marker='x', color='k') - except BaseException: - axarr[rxr - 1].scatter(0, anti + 0.3 * poli[pol], marker='x', color='k') - axarr[rxr - 1].annotate(str(ant) + ',' + pams[ant], xy=[0.01, i]) - axarr[rxr - 1].set_yticks([]) - axarr[rxr - 1].set_xticks([]) -for rxr in range(np.max(rxr_nums)): - axarr[rxr].set_title('Rxr ' + str(rxr + 1)) -plt.xlim([-.01, .1]) -plt.subplots_adjust(bottom=0.15) -plt.subplots_adjust(wspace=0) -# cbar_ax = f.add_axes([.14, .05, .72, .05]) -cbar_ax = fig.add_axes([.13, .05, .67, .05]) -if args.log: - label = '10log10(Median Autos)' -else: - label = 'Median Autos' -fig.colorbar(ax, cax=cbar_ax, orientation='horizontal', label=label) -fig.suptitle(str(latest.datetime) + ' UTC') -# Add polarization key -p_ax = fig.add_axes([0.82, 0.03, 0.06, 0.08]) -for pol in ['xx', 'yy']: - yval = poli[pol] - plt.scatter(0, yval, c=vmax, vmin=vmin, vmax=vmax, cmap=goodbad) - plt.annotate(pol_labels[pol] + ' pol', xy=[1, yval], textcoords='data', verticalalignment='center') -plt.xlim([-1, 4]) -plt.ylim([-1, 2]) -p_ax.set_xticks([]) -p_ax.set_yticks([]) -# Save file -filename = os.path.join(outpath, basename + '.auto_v_rxr.png') -plt.savefig(filename) - -# Plot spectra -fig = plt.figure(figsize=(20, 12)) -# Number of subplots in X and Y directions -nants = len(ants) -nx = int(np.ceil(np.log2(nants + 1))) -ny = int(np.ceil(nants / float(nx))) -pol_colors = {'xx': 'r', 'yy': 'b'} -ants = np.sort(ants) -for ai, ant in enumerate(ants): - ax = plt.subplot(nx, ny, ai + 1) - for pol in ['xx', 'yy']: - try: - plt.plot(autos[(ant, pol)], pol_colors[pol], label=pol_labels[pol]) - except KeyError: - continue - if ant in ants_connected: - lcolor = 'r' - else: - lcolor = 'k' - plt.text(0.8, 0.8, str(ant), fontsize=12, transform=ax.transAxes, color=lcolor) - # Axis stuff - ax.axes.get_xaxis().set_ticklabels([]) - if ai % ny: - ax.axes.get_yaxis().set_ticklabels([]) - else: - if args.log: - ax.set_ylabel('10log10') - else: - ax.set_ylabel('linear') - ax.set_ylim([vmin, 1.3 * vmax]) - if ai == 0: - plt.legend(loc='best') -fig.suptitle(str(latest.datetime) + ' UTC; JD=' + str(latest.jd)) -filename = os.path.join(outpath, basename + '.auto_specs.png') -plt.savefig(filename) - -# Plot rms values at input of ADC's - - -def plot_rms(data, POL='xx'): - """Plots rms values at input to ADC's by looking at the autos. - - Args: - data (dict): Dictionary of auto's. - POL (str): String of polarization.""" - fig = plt.figure(figsize=(20, 12)) - CHUNK = 256 - BINS = 24 - rmscolors = 'bgrcmy' - ants = sorted([i for (i, pp) in data if pp == POL]) - Nrow = np.ceil(np.sqrt(len(ants))) - Mcol = np.ceil(len(ants) / float(Nrow)) - bins = np.logspace(-2, 4, BINS, base=2.) - for cnt, ant in enumerate(ants): - ax = plt.subplot(Nrow, Mcol, cnt + 1) - for ch in xrange(0, 1024, CHUNK): - dchunk = data[ant, POL][ch:ch + CHUNK].flatten() - hist, hbins = np.histogram(np.sqrt(dchunk / 2), bins) - hist = 10**np.log10(hist + .1) - hbins = 0.5 * (hbins[1:] + hbins[:-1]) - ax.fill_between(np.log2(bins), hist, .1, where=hist > .1, color=rmscolors[j], alpha=.5) - bounds = np.where(bins < 2**0, dchunk.size, np.where(bins > 2**2, dchunk.size, 0)) - ax.fill_between(np.log2(bins), bounds, .1, where=bounds > .1, color='black', alpha=.6) - ax.set_yscale('log') - plt.xlim(-2, 3) - plt.ylim(d.size / 1e2, d.size) - if ant in ants_connected: - lcolor = 'r' - else: - lcolor = 'b' - plt.text(0.5, 0.8, str(ant), fontsize=12, transform=ax.transAxes, color=lcolor) - ax.get_yaxis().set_visible(False) - if cnt < (Nrow - 1) * Mcol: - ax.get_xaxis().set_ticklabels([]) - else: - plt.xlabel(r'$V_{\rm rms}$ [bits]') - plt.grid() - fig.suptitle(str(latest.datetime) + ' UTC' + ' {0}'.format(POL) + '; JD=' + str(latest.jd)) - plt.tight_layout(rect=(0, 0, 1, .95)) - - -for pol in ['xx', 'yy']: - if args.log: - plot_rms(autos_raw, POL=pol) - else: - plot_rms(autos, POL=pol) - filename = os.path.join(outpath, basename + '.{0}.auto_rms_values.png'.format(pol)) - plt.savefig(filename) - -# ID some potential baddies -if args.idbaddies: - baddies = [str((key[0], pol_labels[key[1]])) for key, val in amps.items() if - val < 0.75 * (vmax - vmin) + vmin and key[0] in ants_connected] - filename = os.path.join(outpath, basename + '.baddies.txt') - np.savetxt(filename, baddies, fmt='%s', header='You may want to check these antennas:') diff --git a/scripts/delay_xrfi_h1c_idr2_1_run.py b/scripts/delay_xrfi_h1c_idr2_1_run.py deleted file mode 100755 index bb067d4d..00000000 --- a/scripts/delay_xrfi_h1c_idr2_1_run.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -import sys -import numpy as np -from hera_qm import utils as qm_utils -from hera_qm import xrfi -from hera_cal import delay_filter -from hera_cal import io -from pyuvdata import UVData - -ap = qm_utils.get_metrics_ArgumentParser('delay_xrfi_h1c_idr2_1_run') -args = ap.parse_args() -filename = args.filename -history = ' '.join(sys.argv) - -# Read data, apply delay filter, update UVData object -uv = UVData() -uv.read_miriad(filename) -# apply a priori waterfall flags -if args.waterfalls is not None: - waterfalls = args.waterfalls.split(',') - if len(waterfalls) > 0: - xrfi.flag_apply(waterfalls, uv, force_pol=True) - -# set kwargs -kwargs = {} -if args.window == 'tukey': - kwargs['alpha'] = args.alpha - -# Stuff into delay filter object, run delay filter -dfil = delay_filter.Delay_Filter() -dfil.load_data(uv) -dfil.run_filter(standoff=args.standoff, horizon=args.horizon, tol=args.tol, - window=args.window, skip_wgt=args.skip_wgt, maxiter=args.maxiter, **kwargs) -io.update_uvdata(dfil.input_data, data=dfil.filtered_residuals, flags=dfil.flags) - -# Run xrfi -xrfi.xrfi_h1c_run(dfil.input_data, history, infile_format=args.infile_format, - extension=args.extension, summary=args.summary, summary_ext=args.summary_ext, - xrfi_path=args.xrfi_path, model_file=args.model_file, - model_file_format=args.model_file_format, calfits_file=args.calfits_file, - kt_size=args.kt_size, kf_size=args.kf_size, sig_init=args.sig_init, - sig_adj=args.sig_adj, px_threshold=args.px_threshold, - freq_threshold=args.freq_threshold, time_threshold=args.time_threshold, - ex_ants=args.ex_ants, metrics_file=args.metrics_file, filename=args.filename) diff --git a/scripts/firstcal_metrics_run.py b/scripts/firstcal_metrics_run.py deleted file mode 100755 index 6b84fc4a..00000000 --- a/scripts/firstcal_metrics_run.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -from hera_qm import utils -from hera_qm import firstcal_metrics -import sys - -ap = utils.get_metrics_ArgumentParser('firstcal_metrics') -args = ap.parse_args() -files = args.files -history = ' '.join(sys.argv) - -firstcal_metrics.firstcal_metrics_run(files, args, history) diff --git a/scripts/omnical_metrics_run.py b/scripts/omnical_metrics_run.py deleted file mode 100755 index e87d6714..00000000 --- a/scripts/omnical_metrics_run.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -from hera_qm import utils -from hera_qm import omnical_metrics -import sys - -ap = utils.get_metrics_ArgumentParser('omnical_metrics') -args = ap.parse_args() -files = args.files -history = ' '.join(sys.argv) - -omnical_metrics.omnical_metrics_run(files, args, history) diff --git a/scripts/xrfi_apply.py b/scripts/xrfi_apply.py deleted file mode 100755 index 813dcf45..00000000 --- a/scripts/xrfi_apply.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -import sys -from hera_qm import utils -from hera_qm import xrfi - -ap = utils.get_metrics_ArgumentParser('xrfi_apply') -args = ap.parse_args() -filename = args.filename -history = ' '.join(sys.argv) - -xrfi.xrfi_h1c_apply(filename, history, infile_format=args.infile_format, xrfi_path=args.xrfi_path, - outfile_format=args.outfile_format, extension=args.extension, - overwrite=args.overwrite, flag_file=args.flag_file, waterfalls=args.waterfalls, - output_uvflag=args.output_uvflag, output_uvflag_ext=args.output_uvflag_ext) diff --git a/scripts/xrfi_day_threshold_run.py b/scripts/xrfi_day_threshold_run.py deleted file mode 100755 index 99bed8d9..00000000 --- a/scripts/xrfi_day_threshold_run.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -import sys -from hera_qm import utils -from hera_qm import xrfi - -ap = utils.get_metrics_ArgumentParser('day_threshold_run') -args = ap.parse_args() -history = ' '.join(sys.argv) - -if args.run_if_first is None or sorted(args.data_files)[0] == args.run_if_first: - xrfi.day_threshold_run(args.data_files, history, nsig_f=args.nsig_f, nsig_t=args.nsig_t, flag_abscal=not(args.skip_making_flagged_abs_calfits), - nsig_f_adj=args.nsig_f_adj, nsig_t_adj=args.nsig_t_adj, clobber=args.clobber, - a_priori_flag_yaml=args.a_priori_flag_yaml) -else: - print(sorted(args.data_files)[0], 'is not', args.run_if_first, '...skipping.') diff --git a/scripts/xrfi_h1c_run.py b/scripts/xrfi_h1c_run.py deleted file mode 100755 index 077e7e89..00000000 --- a/scripts/xrfi_h1c_run.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -import sys -from hera_qm import utils -from hera_qm import xrfi - -ap = utils.get_metrics_ArgumentParser('xrfi_h1c_run') -args = ap.parse_args() -filename = args.filename -history = ' '.join(sys.argv) - -xrfi.xrfi_h1c_run(filename, history, infile_format=args.infile_format, - extension=args.extension, summary=args.summary, summary_ext=args.summary_ext, - xrfi_path=args.xrfi_path, model_file=args.model_file, - model_file_format=args.model_file_format, calfits_file=args.calfits_file, - kt_size=args.kt_size, kf_size=args.kf_size, sig_init=args.sig_init, - sig_adj=args.sig_adj, px_threshold=args.px_threshold, - freq_threshold=args.freq_threshold, time_threshold=args.time_threshold, - ex_ants=args.ex_ants, metrics_file=args.metrics_file, filename=filename[0]) diff --git a/scripts/xrfi_h3c_idr2_1_run.py b/scripts/xrfi_h3c_idr2_1_run.py deleted file mode 100755 index 2b6f7d6d..00000000 --- a/scripts/xrfi_h3c_idr2_1_run.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2020 the HERA Project -# Licensed under the MIT License - -import sys -from hera_qm import utils -from hera_qm import xrfi - -ap = utils.get_metrics_ArgumentParser('xrfi_h3c_idr2_1_run') -args = ap.parse_args() -flag_command = ' '.join(sys.argv) - -xrfi.xrfi_h3c_idr2_1_run(args.ocalfits_files, args.acalfits_files, args.model_files, - args.data_files, flag_command, xrfi_path=args.xrfi_path, - kt_size=args.kt_size, kf_size=args.kf_size, sig_init=args.sig_init, - sig_adj=args.sig_adj, ex_ants=args.ex_ants, - metrics_file=args.metrics_file, clobber=args.clobber) diff --git a/scripts/xrfi_h3ca_rtp_run.py b/scripts/xrfi_h3ca_rtp_run.py deleted file mode 100755 index 6b7cb462..00000000 --- a/scripts/xrfi_h3ca_rtp_run.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -import sys -from hera_qm import utils -from hera_qm import xrfi - -ap = utils.get_metrics_ArgumentParser('xrfi_run') # Use same parser as IDR2.2 script -args = ap.parse_args() -history = ' '.join(sys.argv) - -xrfi.xrfi_run(args.data_file, history, xrfi_path=args.xrfi_path, - kt_size=args.kt_size, kf_size=args.kf_size, sig_init=args.sig_init, - sig_adj=args.sig_adj, ex_ants=args.ex_ants, - metrics_file=args.metrics_file, clobber=args.clobber) diff --git a/scripts/xrfi_run.py b/scripts/xrfi_run.py deleted file mode 100755 index fcf00cba..00000000 --- a/scripts/xrfi_run.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -import sys -from hera_qm import utils -from hera_qm import xrfi - -ap = utils.get_metrics_ArgumentParser('xrfi_run') -args = ap.parse_args() -history = ' '.join(sys.argv) - -xrfi.xrfi_run(args.ocalfits_files, - args.acalfits_files, - args.model_files, - args.data_files, - a_priori_flag_yaml=args.a_priori_flag_yaml, - a_priori_ants_only=not(args.a_apriori_times_and_freqs), - use_cross_pol_vis=not(args.skip_cross_pol_vis), - history=history, - xrfi_path=args.xrfi_path, - throw_away_edges=not(args.keep_edge_times), - kt_size=args.kt_size, - kf_size=args.kf_size, - sig_init_med=args.sig_init_med, - sig_adj_med=args.sig_adj_med, - sig_init_mean=args.sig_init_mean, - sig_adj_mean=args.sig_adj_mean, - ex_ants=args.ex_ants, - Nwf_per_load=args.Nwf_per_load, - omnical_median_filter=not(args.skip_omnical_median_filter), - omnical_mean_filter=not(args.skip_omnical_mean_filter), - omnical_chi2_median_filter=not(args.skip_omnical_chi2_median_filter), - omnical_chi2_mean_filter=not(args.skip_omnical_chi2_mean_filter), - omnical_zscore_filter=not(args.skip_omnical_zscore_filter), - abscal_median_filter=not(args.skip_abscal_median_filter), - abscal_mean_filter=not(args.skip_abscal_mean_filter), - abscal_chi2_median_filter=not(args.skip_abscal_chi2_median_filter), - abscal_chi2_mean_filter=not(args.skip_abscal_chi2_mean_filter), - abscal_zscore_filter=not(args.skip_abscal_zscore_filter), - omnivis_median_filter=not(args.skip_omnivis_median_filter), - omnivis_mean_filter=not(args.skip_omnivis_mean_filter), - auto_median_filter=not(args.skip_auto_median_filter), - auto_mean_filter=not(args.skip_auto_mean_filter), - cross_median_filter=args.use_cross_median_filter, - cross_mean_filter=not(args.skip_cross_mean_filter), - metrics_files=args.metrics_files, clobber=args.clobber) diff --git a/scripts/xrfi_run_data_only.py b/scripts/xrfi_run_data_only.py deleted file mode 100644 index e520db99..00000000 --- a/scripts/xrfi_run_data_only.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2019 the HERA Project -# Licensed under the MIT License - -import sys -from hera_qm import utils -from hera_qm import xrfi - -ap = utils.get_metrics_ArgumentParser('xrfi_run_data_only') -args = ap.parse_args() -history = ' '.join(sys.argv) - -xrfi.xrfi_run(data_files=args.data_files, - a_priori_flag_yaml=args.a_priori_flag_yaml, - use_cross_pol_vis=not(args.skip_cross_pol_vis), - cross_median_filter=args.cross_median_filter, - cross_mean_filter=not(args.skip_cross_mean_filter), - history=history, - xrfi_path=args.xrfi_path, - throw_away_edges=not(args.keep_edge_times), - kt_size=args.kt_size, - kf_size=args.kf_size, - sig_init_med=args.sig_init_med, - sig_adj_med=args.sig_adj_med, - sig_init_mean=args.sig_init_mean, - sig_adj_mean=args.sig_adj_mean, - ex_ants=args.ex_ants, - Nwf_per_load=args.Nwf_per_load, - metrics_files=args.metrics_files, - clobber=args.clobber) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 4274b292..00000000 --- a/setup.cfg +++ /dev/null @@ -1,13 +0,0 @@ -[aliases] -test=pytest - -[tool:pytest] -addopts = --ignore=scripts --cov=hera_qm --cov-config=.coveragerc - -[pycodestyle] -ignore = E501,W503 -filename = *.py -exclude = build/,docs/,.pytest_cache,.svn,CVS,.bzr,.hg,.git,__pycache__,dist/,scripts/,hera_qm.egg-info,hera_qm/data/ - -[pydocstyle] -convention = numpy diff --git a/setup.py b/setup.py deleted file mode 100644 index 815abf0b..00000000 --- a/setup.py +++ /dev/null @@ -1,67 +0,0 @@ -from setuptools import setup -import os -import sys -import os.path as op -import json -from pathlib import Path - -sys.path.append('hera_qm') - -def package_files(package_dir, subdirectory): - # walk the input package_dir/subdirectory - # return a package_data list - paths = [] - directory = os.path.join(package_dir, subdirectory) - for (path, directories, filenames) in os.walk(directory): - for filename in filenames: - path = path.replace(package_dir + '/', '') - paths.append(os.path.join(path, filename)) - return paths - - -data_files = package_files('hera_qm', 'data') -this_directory = Path(__file__).parent -long_description = (this_directory / "README.md").read_text() - -setup_args = { - 'name': 'hera_qm', - 'author': 'HERA Team', - 'url': 'https://github.com/HERA-Team/hera_qm', - 'license': 'BSD', - 'description': 'HERA Data Quality Metrics.', - 'long_description': long_description, - 'long_description_content_type': 'text/markdown', - 'package_dir': {'hera_qm': 'hera_qm'}, - 'packages': ['hera_qm'], - 'include_package_data': True, - 'scripts': ['scripts/ant_metrics_run.py', - 'scripts/auto_metrics_run.py', - 'scripts/xrfi_run.py', - 'scripts/firstcal_metrics_run.py', - 'scripts/auto_view.py', - 'scripts/omnical_metrics_run.py', - 'scripts/xrfi_apply.py', - 'scripts/delay_xrfi_h1c_idr2_1_run.py', - 'scripts/xrfi_h1c_run.py', - 'scripts/xrfi_day_threshold_run.py', - 'scripts/xrfi_h3c_idr2_1_run.py', - 'scripts/xrfi_h3ca_rtp_run.py', - 'scripts/xrfi_run_data_only.py' - ], - 'package_data': {'hera_qm': data_files}, - 'setup_requires': ['pytest-runner'], - 'install_requires': [ - 'astropy>=5.0.4', - 'h5py>=3.1', - 'pyyaml', - 'numpy>=1.23', - 'pyuvdata>=2.3', - 'hera_filters', - ], - 'tests_require': ['pytest'], - 'zip_safe': False, -} - - -if __name__ == '__main__': - setup(**setup_args)