From 910ad21a07aae8c938796215a9803bed0966b2ac Mon Sep 17 00:00:00 2001 From: Steven Murray Date: Tue, 9 Jul 2024 03:03:58 +0200 Subject: [PATCH 1/4] fix: update everything to use pyuvdata 3.0 --- hera_qm/firstcal_metrics.py | 5 +- hera_qm/metrics_io.py | 24 ++-- hera_qm/omnical_metrics.py | 12 +- hera_qm/tests/test_ant_metrics.py | 6 +- hera_qm/tests/test_auto_metrics.py | 3 + hera_qm/tests/test_utils.py | 7 +- hera_qm/tests/test_vis_metrics.py | 13 +-- hera_qm/tests/test_xrfi.py | 123 +++++++++------------ hera_qm/utils.py | 7 -- hera_qm/xrfi.py | 171 +++++++++-------------------- pyproject.toml | 2 +- 11 files changed, 137 insertions(+), 236 deletions(-) diff --git a/hera_qm/firstcal_metrics.py b/hera_qm/firstcal_metrics.py index c77204d9..681d01bf 100644 --- a/hera_qm/firstcal_metrics.py +++ b/hera_qm/firstcal_metrics.py @@ -345,10 +345,7 @@ def __init__(self, calfits_files, use_gp=True): # Instantiate UVCal and read calfits self.UVC = UVCal() with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "Future array shapes are now always used" - ) - self.UVC.read_calfits(calfits_files, use_future_array_shapes=True) + self.UVC.read_calfits(calfits_files) if hasattr(self.UVC, "telescope"): x_orientation = self.UVC.telescope.x_orientation diff --git a/hera_qm/metrics_io.py b/hera_qm/metrics_io.py index 78461b38..e7f17f70 100644 --- a/hera_qm/metrics_io.py +++ b/hera_qm/metrics_io.py @@ -117,8 +117,15 @@ def _recursively_save_dict_to_group(h5file, path, in_dict): """ allowed_types = (np.ndarray, np.float32, float, int, bytes, str, list, bool, np.bool_) + + compressable_types = (np.ndarray, list) for key in in_dict: + # Catch case with numpy 2 where keys of type (np.int32(ant), 'polstr')) are + # converted to string with the full np.int32 identifier in them. + if isinstance(key, tuple): + key = tuple(int(k) if isinstance(k, (int, np.int32, np.int64)) else k for k in key) + key_str = str(key) if key == 'reds': in_dict[key] = _reds_list_to_dict(in_dict[key]) @@ -346,17 +353,16 @@ def _recursively_load_dict_to_group(h5file, path, group_is_ordered=False): item = h5file[path][key] - if isinstance(item, h5py.Dataset): - if item.attrs['key_is_string']: - out_dict[str(key)] = item[()] - else: - out_key = _parse_key(key) + if isinstance(item, (h5py.Dataset, h5py.Group)): + out_key = str(key) if item.attrs['key_is_string'] else _parse_key(key) + #print(key, out_key, item.attrs['key_is_string']) + if isinstance(item, h5py.Dataset): out_dict[out_key] = item[()] - elif isinstance(item, h5py.Group): - out_key = str(key) if item.attrs['key_is_string'] else _parse_key(key) - out_dict[out_key] = _recursively_load_dict_to_group(h5file, (path + key + '/'), - group_is_ordered=item.attrs["group_is_ordered"]) + elif isinstance(item, h5py.Group): + out_dict[out_key] = _recursively_load_dict_to_group( + h5file, (path + key + '/'), group_is_ordered=item.attrs["group_is_ordered"] + ) else: raise TypeError("The HDF5 path: {} is not associated with either " "a dataset or group object. " diff --git a/hera_qm/omnical_metrics.py b/hera_qm/omnical_metrics.py index 6161bb4e..ced1dc9a 100644 --- a/hera_qm/omnical_metrics.py +++ b/hera_qm/omnical_metrics.py @@ -174,11 +174,7 @@ def load_firstcal_gains(fc_file): The polarization of the .calfits file. """ uvc = UVCal() - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "Future array shapes are now always used" - ) - uvc.read_calfits(fc_file, use_future_array_shapes=True) + uvc.read_calfits(fc_file) fc_gains = np.moveaxis(uvc.gain_array, 1, 2)[:, :, :, 0] d_nu = np.mean(uvc.freq_array[1:] - uvc.freq_array[:-1]) d_phi = np.abs(np.mean(np.angle(fc_gains)[:, :, 1:] - np.angle(fc_gains)[:, :, :-1], axis=2)) @@ -453,11 +449,7 @@ def __init__(self, omni_calfits, history=''): # Instantiate Data Object self.uv = UVCal() - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "Future array shapes are now always used" - ) - self.uv.read_calfits(omni_calfits, use_future_array_shapes=True) + self.uv.read_calfits(omni_calfits) # Get relevant metadata self.Nants = self.uv.Nants_data diff --git a/hera_qm/tests/test_ant_metrics.py b/hera_qm/tests/test_ant_metrics.py index 1b447ec9..ac90cda4 100644 --- a/hera_qm/tests/test_ant_metrics.py +++ b/hera_qm/tests/test_ant_metrics.py @@ -364,7 +364,11 @@ def test_ant_metrics_run_and_load_antenna_metrics(): # test a priori flagging via YAML four_pol_uvh5 = DATA_PATH + '/zen.2457698.40355.full_pol_test.uvh5' apf_yaml = os.path.join(DATA_PATH, 'a_priori_flags_sample.yaml') - am = ant_metrics.ant_metrics_run(four_pol_uvh5, diff_files=four_pol_uvh5, overwrite=True, a_priori_xants_yaml=apf_yaml, verbose=True) + + # SGM: not sure if the following warning *should* be raised. + with pytest.warns(RuntimeWarning, match='All-NaN slice encountered'): + am = ant_metrics.ant_metrics_run(four_pol_uvh5, diff_files=four_pol_uvh5, overwrite=True, a_priori_xants_yaml=apf_yaml, verbose=True) + am_hdf5 = ant_metrics.load_antenna_metrics(four_pol_uvh5.replace('.uvh5', '.ant_metrics.hdf5')) for ant in [(0, 'Jee'), (0, 'Jnn'), (10, 'Jee'), (10, 'Jnn'), (1, 'Jee'), (3, 'Jnn')]: assert ant in am_hdf5['xants'] diff --git a/hera_qm/tests/test_auto_metrics.py b/hera_qm/tests/test_auto_metrics.py index 6560382e..e0406a2f 100644 --- a/hera_qm/tests/test_auto_metrics.py +++ b/hera_qm/tests/test_auto_metrics.py @@ -219,6 +219,9 @@ def test_auto_metrics_run(): with warnings.catch_warnings(): warnings.filterwarnings("ignore", message="All-NaN slice encountered") warnings.filterwarnings("ignore", message="K1 value 8 is larger than the data of dimension 4; using the size of the data for the kernel size") + warnings.filterwarnings("ignore", message="Mean of empty slice") + warnings.filterwarnings("ignore", message="Degrees of freedom <= 0 for slice") + ex_ants, modzs, spectra, flags = auto_metrics.auto_metrics_run(metrics_outfile, auto_files, median_round_modz_cut=75., mean_round_modz_cut=5., edge_cut=100, Kt=8, Kf=8, sig_init=5.0, sig_adj=2.0, diff --git a/hera_qm/tests/test_utils.py b/hera_qm/tests/test_utils.py index c8681a39..70b2fe12 100644 --- a/hera_qm/tests/test_utils.py +++ b/hera_qm/tests/test_utils.py @@ -286,7 +286,7 @@ def test_apply_yaml_flags_uvdata(tmpdir, filein, flag_freqs, flag_times, flag_an integration_flags = [0, 1] # integrations from yaml file that should be flagged. ant_flags = [0, 10, [1, 'Jee'], [3, 'Jnn']] uvd = UVData() - uvd.read(test_d_file, use_future_array_shapes=True) + uvd.read(test_d_file) uvd = utils.apply_yaml_flags(uvd, test_flag, flag_freqs=flag_freqs, flag_times=flag_times, flag_ants=flag_ants, unflag_first=True) if 'no_integrations' not in test_flag: @@ -368,7 +368,7 @@ def test_apply_yaml_flags_uvcal(filein): test_flag = os.path.join(DATA_PATH, filein) test_c_file = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA.omni.calfits') uvc = UVCal() - uvc.read_calfits(test_c_file, use_future_array_shapes=True) + uvc.read_calfits(test_c_file) uvc = utils.apply_yaml_flags(uvc, test_flag, unflag_first=True) freq_regions = [(0, 110e6), (150e6, 155e6), (190e6, 200e6)] # frequencies from yaml file. @@ -420,13 +420,12 @@ def test_apply_yaml_flags_uvcal(filein): throw_away_flagged_ants=True, ant_indices_only=False) -@pytest.mark.filterwarnings("ignore:Future array shapes are now always used") def test_apply_yaml_flags_errors(): test_flag_jds = os.path.join(DATA_PATH, 'a_priori_flags_jds.yaml') test_c_file = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA.omni.calfits') # check NotImplementedErrors uvc = UVCal() - uvc.read_calfits(test_c_file, use_future_array_shapes=True) + uvc.read_calfits(test_c_file) # check that setting uv to an object that is not a subclass of UVCal or UVData throws a NotImplementedError pytest.raises(NotImplementedError, utils.apply_yaml_flags, 'uvdata', test_flag_jds) diff --git a/hera_qm/tests/test_vis_metrics.py b/hera_qm/tests/test_vis_metrics.py index b3ffc743..05e3705e 100644 --- a/hera_qm/tests/test_vis_metrics.py +++ b/hera_qm/tests/test_vis_metrics.py @@ -10,6 +10,7 @@ import copy import pytest import warnings +from erfa.core import ErfaWarning pytestmark = pytest.mark.filterwarnings( "ignore:The uvw_array does not match the expected values given the antenna positions.", @@ -19,9 +20,7 @@ def vismetrics_data(): data = UVData() filename = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA') - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - data.read_miriad(filename, use_future_array_shapes=True, check_autos=False) + data.read_miriad(filename, check_autos=False) # massage the object to make it work with check_noise_variance data.select(antenna_nums=data.get_ants()[0:10]) @@ -30,7 +29,8 @@ def vismetrics_data(): while data.Ntimes < 90: d2 = copy.deepcopy(data) d2.time_array += d2.time_array.max() + d2.integration_time / (24 * 3600) - data += d2 + with pytest.raises(ErfaWarning, match='ERFA function'): + data += d2 ntimes = data.Ntimes nchan = data.Nfreqs data1 = qmtest.noise(size=(ntimes, nchan)) @@ -79,10 +79,9 @@ def test_check_noise_variance_inttime_error(vismetrics_data): def uvd(): uvd = UVData() with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") uvd.read_miriad( os.path.join(DATA_PATH, 'zen.2458002.47754.xx.HH.uvA'), - projected=False, use_future_array_shapes=True, + projected=False, check_autos=False ) return uvd @@ -155,7 +154,7 @@ def test_plot_bl_bl_scatter(uvd): @pytest.mark.filterwarnings("ignore:Future array shapes are now always used") def test_sequential_diff(): uvd = UVData() - uvd.read_miriad(os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA'), use_future_array_shapes=True) + uvd.read_miriad(os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA')) # diff across time uvd_diff = vis_metrics.sequential_diff(uvd, axis=0, pad=False) diff --git a/hera_qm/tests/test_xrfi.py b/hera_qm/tests/test_xrfi.py index 986b0bb3..cd4599c2 100644 --- a/hera_qm/tests/test_xrfi.py +++ b/hera_qm/tests/test_xrfi.py @@ -56,9 +56,7 @@ @pytest.fixture(scope="session") def uvdata_miriad_main(): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uv = UVData.from_file(test_d_file, use_future_array_shapes=True) + uv = UVData.from_file(test_d_file) yield uv # clean up when done @@ -80,11 +78,7 @@ def uvdata_miriad(uvdata_miriad_main): @pytest.fixture(scope="session") def uvflag_metric_miriad_main(uvdata_miriad_main): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvm = UVFlag( - uvdata_miriad_main, history='I made this', use_future_array_shapes=True - ) + uvm = UVFlag(uvdata_miriad_main, history='I made this') yield uvm # clean up when done del uvm @@ -104,14 +98,11 @@ def uvflag_metric_miriad(uvflag_metric_miriad_main): @pytest.fixture(scope="session") def uvflag_metric_waterfall_miriad_main(uvdata_miriad_main): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvm = UVFlag( - uvdata_miriad_main, - history='I made this', - waterfall=True, - use_future_array_shapes=True - ) + uvm = UVFlag( + uvdata_miriad_main, + history='I made this', + waterfall=True, + ) yield uvm # clean up when done del uvm @@ -131,9 +122,7 @@ def uvflag_metric_waterfall_miriad(uvflag_metric_waterfall_miriad_main): @pytest.fixture(scope="session") def uvflag_flag_miriad_main(uvdata_miriad_main): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf = UVFlag(uvdata_miriad_main, mode='flag', use_future_array_shapes=True) + uvf = UVFlag(uvdata_miriad_main, mode='flag') yield uvf # clean up when done del uvf @@ -153,14 +142,11 @@ def uvflag_flag_miriad(uvflag_flag_miriad_main): @pytest.fixture(scope="session") def uvflag_flag_waterfall_miriad_main(uvdata_miriad_main): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf = UVFlag( - uvdata_miriad_main, - mode='flag', - waterfall=True, - use_future_array_shapes=True - ) + uvf = UVFlag( + uvdata_miriad_main, + mode='flag', + waterfall=True, + ) yield uvf # clean up when done del uvf @@ -180,9 +166,7 @@ def uvflag_flag_waterfall_miriad(uvflag_flag_waterfall_miriad_main): @pytest.fixture(scope="session") def uvcal_calfits_main(): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvc = UVCal.from_file(test_c_file, use_future_array_shapes=True) + uvc = UVCal.from_file(test_c_file) yield uvc # clean up when done @@ -204,11 +188,7 @@ def uvcal_calfits(uvcal_calfits_main): @pytest.fixture(scope="session") def uvflag_metric_calfits_main(uvcal_calfits_main): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvm = UVFlag( - uvcal_calfits_main, history='I made this', use_future_array_shapes=True - ) + uvm = UVFlag(uvcal_calfits_main, history='I made this') yield uvm # clean up when done @@ -230,9 +210,7 @@ def uvflag_metric_calfits(uvflag_metric_calfits_main): @pytest.fixture(scope="session") def uvflag_flag_calfits_main(uvcal_calfits_main): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf = UVFlag(uvcal_calfits_main, mode='flag', use_future_array_shapes=True) + uvf = UVFlag(uvcal_calfits_main, mode='flag') yield uvf # clean up when done @@ -254,9 +232,7 @@ def uvflag_flag_calfits(uvflag_flag_calfits_main): @pytest.fixture(scope="session") def uvflag_f_main(): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf = UVFlag(test_f_file, use_future_array_shapes=True) + uvf = UVFlag(test_f_file) yield uvf # clean up when done @@ -427,6 +403,7 @@ def test_robus_divide(): assert np.array_equal(c, np.array([1. / 2., np.inf, np.inf])) +@pytest.mark.filterwarnings("ignore:Casting complex values to real") def test_dpss_flagger(): freqs = np.linspace(50e6, 250e6, 500) df = np.diff(freqs)[0] @@ -1173,7 +1150,7 @@ def test_flag_apply( uv = uvcal_calfits uv.flag_array = np.zeros_like(uv.flag_array, dtype=np.bool_) - uvf = UVFlag(uv, mode='flag', waterfall=True, use_future_array_shapes=True) + uvf = UVFlag(uv, mode='flag', waterfall=True) uvf.flag_array[:, 0, :] = True xrfi.flag_apply(uvf, uv) assert np.allclose(uv.flag_array[:, 0, :, :], True) @@ -1421,7 +1398,9 @@ def test_xrfi_run_step(tmpdir): assert np.all(np.isclose(uvf_a1.flag_array, uvf_a2.flag_array)) assert np.all(np.isclose(uvf1.metric_array, uvf2.metric_array, atol=1e-5, rtol=1e-5)) # hit one line involving uvcal reinitialization and include yaml file. - xrfi.xrfi_run_step(uv_files=ocal_file, calculate_uvf_apriori=True, run_filter=True, reinitialize=True, a_priori_flag_yaml=a_priori_flag_integrations) + + with pytest.warns(RuntimeWarning, match="invalid value encountered in subtract"): + xrfi.xrfi_run_step(uv_files=ocal_file, calculate_uvf_apriori=True, run_filter=True, reinitialize=True, a_priori_flag_yaml=a_priori_flag_integrations) # test invalid data type error with pytest.raises(ValueError): xrfi.xrfi_run_step(uv_files=ocal_file, calculate_uvf_apriori=True, run_filter=True, reinitialize=True, dtype='uvwhatever') @@ -1515,7 +1494,7 @@ def test_xrfi_run_yaml_flags(tmpdir): if ext not in ['cross_metrics1', 'cross_flags1']: out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # check that all flags in frequency regions are set to True. # only check init flags, apriori flags, and round 2 flags. @@ -1603,7 +1582,7 @@ def test_xrfi_run(tmpdir): if ext not in ['cross_metrics1', 'cross_flags1']: out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # cleanup for ext in ext_labels: @@ -1620,7 +1599,7 @@ def test_xrfi_run(tmpdir): for ext, label in ext_labels.items(): out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) uvf_list1.append(uvf) uvf_list1_names.append(out) assert uvf.label == label @@ -1638,7 +1617,7 @@ def test_xrfi_run(tmpdir): for ext, label in ext_labels.items(): out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) uvf_list2.append(uvf) uvf_list2_names.append(out) assert uvf.label == label @@ -1665,7 +1644,7 @@ def test_xrfi_run(tmpdir): out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) if 'cross' in ext or 'combined' in ext: assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # cleanup for ext in ext_labels: @@ -1679,7 +1658,7 @@ def test_xrfi_run(tmpdir): out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) if 'auto' in ext or 'combined' in ext: assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # cleanup for ext in ext_labels: @@ -1699,7 +1678,7 @@ def test_xrfi_run(tmpdir): out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) if 'cross' not in ext and 'v_' not in ext and 'auto' not in ext: assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # cleanup for ext in ext_labels: @@ -1714,7 +1693,7 @@ def test_xrfi_run(tmpdir): if 'cross' not in ext and 'v_' not in ext and 'auto' not in ext\ and 'ox_' not in ext and 'og_' not in ext and 'omnical' not in ext: assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # cleanup for ext in ext_labels: @@ -1727,7 +1706,7 @@ def test_xrfi_run(tmpdir): out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) if 'cross' in ext or 'combined' in ext or 'auto' in ext: assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # cleanup for ext in ext_labels: @@ -1749,7 +1728,7 @@ def test_xrfi_run(tmpdir): out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) if 'cross' in ext or 'combined' in ext or 'auto' in ext: assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # cleanup for ext in ext_labels: @@ -1772,7 +1751,7 @@ def test_xrfi_run(tmpdir): out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) if ('cross' in ext or 'combined' in ext or 'auto' in ext) and '1' not in ext: assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # cleanup for ext in ext_labels: @@ -1910,7 +1889,7 @@ def test_xrfi_run_edgeflag(tmpdir): if ext not in ['cross_metrics1', 'cross_flags1']: out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) if ext == 'flags2.h5': # check that two within edges are flagged assert np.all(uvf.flag_array[:2]) @@ -1949,7 +1928,7 @@ def test_xrfi_run_edgeflag(tmpdir): xrfi.xrfi_run(ocal_files, acal_files, model_files, raw_dfiles, history='Just a test', kt_size=1) flags2 = sorted(glob.glob(tmp_path + '/*.xrfi/*.HH.flags2.h5')) assert len(flags2) == 3 - uvf = UVFlag(flags2, use_future_array_shapes=True) + uvf = UVFlag(flags2) # check that two within edges are flagged assert np.all(uvf.flag_array[:1]) assert np.all(uvf.flag_array[-1:]) @@ -2041,7 +2020,7 @@ def test_xrfi_run_multifile(tmpdir): # by default, only cross median filter / mean filter is not performed. out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # all of flags2 should be flagged with this kt_size. if ext == 'flags2.h5': @@ -2060,7 +2039,7 @@ def test_xrfi_run_multifile(tmpdir): # by default, only cross median filter / mean filter is not performed. out = os.path.join(outdir, '.'.join([fake_obs, ext, 'h5'])) assert os.path.exists(out) - uvf = UVFlag(out, use_future_array_shapes=True) + uvf = UVFlag(out) assert uvf.label == label # if we don't throw away the edges, then there shouldn't be flags # at the edge. @@ -2090,7 +2069,7 @@ def test_day_threshold_run(tmpdir): xrfi.xrfi_run(ocal_file, acal_file, model_file, raw_dfile, history='Just a test', kt_size=3, throw_away_edges=False) # Need to adjust time arrays when duplicating files - uvd = UVData.from_file(data_files[0], use_future_array_shapes=True) + uvd = UVData.from_file(data_files[0]) dt = (uvd.time_array.max() - uvd.time_array.min()) + uvd.integration_time.mean() / (24. * 3600.) uvd.time_array += dt uvd.set_lsts_from_time_array() @@ -2098,7 +2077,7 @@ def test_day_threshold_run(tmpdir): uvd.write_uvh5(data_files[1]) model_file = os.path.join(tmp_path, fake_obses[1] + '.omni_vis.uvh5') uvd.write_uvh5(model_file) - uvc = UVCal.from_file(ocal_file, use_future_array_shapes=True) + uvc = UVCal.from_file(ocal_file) dt = (uvc.time_array.max() - uvc.time_array.min()) + uvc.integration_time.mean() / (24. * 3600.) uvc.time_array += dt uvc.set_lsts_from_time_array() @@ -2160,7 +2139,7 @@ def test_day_threshold_run_yaml(tmpdir): xrfi.xrfi_run(ocal_file, acal_file, model_file, raw_dfile, history='Just a test', kt_size=3, throw_away_edges=False) # Need to adjust time arrays when duplicating files - uvd = UVData.from_file(data_files[0], use_future_array_shapes=True) + uvd = UVData.from_file(data_files[0]) dt = (uvd.time_array.max() - uvd.time_array.min()) + uvd.integration_time.mean() / (24. * 3600.) uvd.time_array += dt uvd.set_lsts_from_time_array() @@ -2168,7 +2147,7 @@ def test_day_threshold_run_yaml(tmpdir): uvd.write_uvh5(data_files[1]) model_file = os.path.join(tmp_path, fake_obses[1] + '.omni_vis.uvh5') uvd.write_uvh5(model_file) - uvc = UVCal.from_file(ocal_file, use_future_array_shapes=True) + uvc = UVCal.from_file(ocal_file) dt = (uvc.time_array.max() - uvc.time_array.min()) + uvc.integration_time.mean() / (24. * 3600.) uvc.time_array += dt uvc.set_lsts_from_time_array() @@ -2216,7 +2195,7 @@ def test_day_threshold_run_data_only(tmpdir): shutil.copyfile(test_uvh5_file, model_file) xrfi.xrfi_run(None, None, None, raw_dfile, history='Just a test', kt_size=3, throw_away_edges=False) # Need to adjust time arrays when duplicating files - uvd = UVData.from_file(data_files[0], use_future_array_shapes=True) + uvd = UVData.from_file(data_files[0]) dt = (uvd.time_array.max() - uvd.time_array.min()) + uvd.integration_time.mean() / (24. * 3600.) uvd.time_array += dt uvd.set_lsts_from_time_array() @@ -2224,7 +2203,7 @@ def test_day_threshold_run_data_only(tmpdir): uvd.write_uvh5(data_files[1]) model_file = os.path.join(tmp_path, fake_obses[1] + '.omni_vis.uvh5') uvd.write_uvh5(model_file) - uvc = UVCal.from_file(ocal_file, use_future_array_shapes=True) + uvc = UVCal.from_file(ocal_file) dt = (uvc.time_array.max() - uvc.time_array.min()) + uvc.integration_time.mean() / (24. * 3600.) uvc.time_array += dt uvc.set_lsts_from_time_array() @@ -2279,7 +2258,7 @@ def test_day_threshold_run_cal_only(tmpdir): ) # Need to adjust time arrays when duplicating files - uvd = UVData.from_file(data_files[0], use_future_array_shapes=True) + uvd = UVData.from_file(data_files[0]) dt = (uvd.time_array.max() - uvd.time_array.min()) + uvd.integration_time.mean() / (24. * 3600.) uvd.time_array += dt uvd.set_lsts_from_time_array() @@ -2287,7 +2266,7 @@ def test_day_threshold_run_cal_only(tmpdir): uvd.write_uvh5(data_files[1]) model_file = os.path.join(tmp_path, fake_obses[1] + '.omni_vis.uvh5') uvd.write_uvh5(model_file) - uvc = UVCal.from_file(ocal_file, use_future_array_shapes=True) + uvc = UVCal.from_file(ocal_file) dt = (uvc.time_array.max() - uvc.time_array.min()) + uvc.integration_time.mean() / (24. * 3600.) uvc.time_array += dt uvc.set_lsts_from_time_array() @@ -2343,7 +2322,7 @@ def test_day_threshold_run_omnivis_only(tmpdir): shutil.copyfile(test_uvh5_file, model_file) xrfi.xrfi_run(None, None, model_file, None, history='Just a test', kt_size=3, output_prefixes=raw_dfile, throw_away_edges=False) # Need to adjust time arrays when duplicating files - uvd = UVData.from_file(data_files[0], use_future_array_shapes=True) + uvd = UVData.from_file(data_files[0]) dt = (uvd.time_array.max() - uvd.time_array.min()) + uvd.integration_time.mean() / (24. * 3600.) uvd.time_array += dt uvd.set_lsts_from_time_array() @@ -2351,7 +2330,7 @@ def test_day_threshold_run_omnivis_only(tmpdir): uvd.write_uvh5(data_files[1]) model_file = os.path.join(tmp_path, fake_obses[1] + '.omni_vis.uvh5') uvd.write_uvh5(model_file) - uvc = UVCal.from_file(ocal_file, use_future_array_shapes=True) + uvc = UVCal.from_file(ocal_file) dt = (uvc.time_array.max() - uvc.time_array.min()) + uvc.integration_time.mean() / (24. * 3600.) uvc.time_array += dt uvc.set_lsts_from_time_array() @@ -2708,7 +2687,7 @@ def test_xrfi_h3c_idr2_1_run(tmp_path, uvcal_calfits): uvc.write_calfits(ocalfits_files[obsi]) uvc.write_calfits(acalfits_files[obsi]) - uv = UVData.from_file(test_uvh5_file, use_future_array_shapes=True) + uv = UVData.from_file(test_uvh5_file) uv.time_array += obsi * dt uv.set_lsts_from_time_array() uv.write_uvh5(model_files[obsi]) @@ -2762,7 +2741,7 @@ def test_xrfi_h3c_idr2_1_run(tmp_path, uvcal_calfits): for ext, label in ext_labels.items(): out = outdir / '.'.join([obs, ext, 'h5']) assert os.path.exists(out) - uvf = UVFlag(str(out), use_future_array_shapes=True) + uvf = UVFlag(str(out)) assert uvf.label == label shutil.rmtree(outdir) # cleanup @@ -2780,7 +2759,7 @@ def test_xrfi_h3c_idr2_1_run(tmp_path, uvcal_calfits): for ext, label in ext_labels.items(): out = outdir / '.'.join([obs, ext, 'h5']) assert os.path.exists(out) - uvf = UVFlag(str(out), use_future_array_shapes=True) + uvf = UVFlag(str(out)) assert uvf.label == label shutil.rmtree(outdir) # cleanup @@ -2798,7 +2777,7 @@ def test_xrfi_h3c_idr2_1_run(tmp_path, uvcal_calfits): for ext, label in ext_labels.items(): out = outdir / '.'.join([obs, ext, 'h5']) assert os.path.exists(out) - uvf = UVFlag(str(out), use_future_array_shapes=True) + uvf = UVFlag(str(out)) assert uvf.label == label shutil.rmtree(outdir) # cleanup diff --git a/hera_qm/utils.py b/hera_qm/utils.py index 85b2f0ef..69269c3c 100644 --- a/hera_qm/utils.py +++ b/hera_qm/utils.py @@ -733,13 +733,6 @@ def apply_yaml_flags(uv, a_priori_flag_yaml, lat_lon_alt_degrees=None, telescope if not issubclass(uv.__class__, (UVData, UVCal, UVFlag)): raise NotImplementedError("uv must be a UVData, UVCal, or UVFlag object.") - # this is a no-op if it's already using future shapes - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "Future array shapes are now always used" - ) - uv.use_future_array_shapes() - # if UVCal provided and lst_array is None, get lst_array from times. # If lat_lon_alt is not specified, try to infer it from the telescope name, which calfits files generally carry around if unflag_first: diff --git a/hera_qm/xrfi.py b/hera_qm/xrfi.py index 13520abf..a7b474a6 100644 --- a/hera_qm/xrfi.py +++ b/hera_qm/xrfi.py @@ -80,9 +80,7 @@ def flag_xants(uv, xants, inplace=True, run_check=True, uvo.to_flag(run_check=run_check, check_extra=check_extra, run_check_acceptability=run_check_acceptability) else: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvo = UVFlag(uv, mode='flag', use_future_array_shapes=True) + uvo = UVFlag(uv, mode='flag') else: uvo = uv @@ -1060,13 +1058,6 @@ def flag(uvf_m, nsig_p=6., nsig_f=None, nsig_t=None, avg_method='quadmean', if (not isinstance(uvf_m, UVFlag)) or (uvf_m.mode != 'metric'): raise ValueError('uvf_m must be UVFlag instance with mode == "metric."') - # use future array shapes - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", "Future array shapes are now always used" - ) - uvf_m.use_future_array_shapes() - # initialize uvf_f = uvf_m.copy() uvf_f.to_flag(run_check=run_check, check_extra=check_extra, @@ -1258,20 +1249,15 @@ def flag_apply(uvf, uv, keep_existing=True, force_pol=False, history='', raise ValueError('Flags can only be applied to UVData or UVCal objects.') if not isinstance(uvf, (list, tuple, np.ndarray)): uvf = [uvf] - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - net_flags = UVFlag( - uv, - mode='flag', - copy_flags=keep_existing, - history=history, - use_future_array_shapes=True - ) + net_flags = UVFlag( + uv, + mode='flag', + copy_flags=keep_existing, + history=history, + ) for uvf_i in uvf: if isinstance(uvf_i, str): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf_i = UVFlag(uvf_i, use_future_array_shapes=True) # Read file + uvf_i = UVFlag(uvf_i) # Read file elif not isinstance(uvf_i, UVFlag): raise ValueError('Input to apply_flag must be UVFlag or path to UVFlag file.') if uvf_i.mode != 'flag': @@ -1410,9 +1396,7 @@ def calculate_metric(uv, algorithm, cal_mode='gain', run_check=True, alg_func = algorithm_dict[algorithm] except KeyError: raise KeyError('Algorithm not found in list of available functions.') - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf = UVFlag(uv, use_future_array_shapes=True) + uvf = UVFlag(uv) if issubclass(uv.__class__, UVData): uvf.weights_array = uv.nsample_array * np.logical_not(uv.flag_array).astype(np.float64) else: @@ -1877,18 +1861,14 @@ def xrfi_run_step(uv_files=None, uv=None, uvf_apriori=None, if dtype=='uvcal': uv = UVCal() # No partial i/o for uvcal yet. - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - getattr(uv, uvcal_read_method)(uv_files, use_future_array_shapes=True) + getattr(uv, uvcal_read_method)(uv_files) if a_priori_flag_yaml is not None: uv = qm_utils.apply_yaml_flags(uv, a_priori_flag_yaml, flag_ants=not(ignore_xants_override), flag_times=not(a_priori_ants_only), flag_freqs=not(a_priori_ants_only)) elif dtype=='uvdata': - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uv = UVData.from_file(uv_files, read_data=False, use_future_array_shapes=True) + uv = UVData.from_file(uv_files, read_data=False) else: raise ValueError("%s is an invalid dtype. Must be 'uvcal' or 'uvdata'."%dtype) no_uvf_apriori = (uvf_apriori is None) @@ -1897,13 +1877,9 @@ def xrfi_run_step(uv_files=None, uv=None, uvf_apriori=None, # if we want, we can reinitialize uv if reinitialize and uv_files is not None: if issubclass(uv.__class__, UVData): - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uv.read(uv_files, read_data=False, use_future_array_shapes=True) + uv.read(uv_files, read_data=False) else: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - getattr(uv, uvcal_read_method)(uv_files, use_future_array_shapes=True) + getattr(uv, uvcal_read_method)(uv_files) if a_priori_flag_yaml is not None: uv = qm_utils.apply_yaml_flags(uv, a_priori_flag_yaml, flag_ants=not(ignore_xants_override), @@ -1929,14 +1905,11 @@ def xrfi_run_step(uv_files=None, uv=None, uvf_apriori=None, # iterate over baseline chunks for loadnum in range(nloads): # read in chunk - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uv.read( - uv_files, - bls=bls[loadnum * Nwf_per_load:(loadnum + 1) * Nwf_per_load], - axis='blt', - use_future_array_shapes=True - ) + uv.read( + uv_files, + bls=bls[loadnum * Nwf_per_load:(loadnum + 1) * Nwf_per_load], + axis='blt', + ) if a_priori_flag_yaml is not None: uv = qm_utils.apply_yaml_flags(uv, a_priori_flag_yaml, flag_times=not(a_priori_ants_only), @@ -1946,15 +1919,12 @@ def xrfi_run_step(uv_files=None, uv=None, uvf_apriori=None, # and we want to calculate it if calculate_uvf_apriori: # then extract the flags for the chunk of baselines we are on - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf_apriori_chunk = UVFlag( - uv, - mode='flag', - copy_flags=True, - label='A priori flags.', - use_future_array_shapes=True - ) + uvf_apriori_chunk = UVFlag( + uv, + mode='flag', + copy_flags=True, + label='A priori flags.', + ) # waterfall them uvf_apriori_chunk.to_waterfall(method='and', keep_pol=False, run_check=run_check, check_extra=check_extra, @@ -2006,15 +1976,12 @@ def xrfi_run_step(uv_files=None, uv=None, uvf_apriori=None, # do so here. if uvf_apriori is None: if calculate_uvf_apriori: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf_apriori = UVFlag( - uv, - mode='flag', - copy_flags=True, - label='A priori flags.', - use_future_array_shapes=True - ) + uvf_apriori = UVFlag( + uv, + mode='flag', + copy_flags=True, + label='A priori flags.', + ) uvf_apriori.to_waterfall(method='and', keep_pol=False, run_check=run_check, check_extra=check_extra, run_check_acceptability=run_check_acceptability) @@ -2470,26 +2437,18 @@ def _run_all_filters(median_round, # Read metadata from first file to get integrations per file. if data_files is not None: uvlist = data_files - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvtemp = UVData.from_file(uvlist[0], read_data=False, use_future_array_shapes=True) + uvtemp = UVData.from_file(uvlist[0], read_data=False) elif model_files is not None: uvlist = model_files - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvtemp = UVData.from_file(uvlist[0], read_data=False, use_future_array_shapes=True) + uvtemp = UVData.from_file(uvlist[0], read_data=False) elif ocalfits_files is not None: uvlist = ocalfits_files uvtemp = UVCal() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - getattr(uvtemp, uvcal_read_method)(uvlist[0], use_future_array_shapes=True) + getattr(uvtemp, uvcal_read_method)(uvlist[0]) elif acalfits_files is not None: uvlist = acalfits_files uvtemp = UVCal() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - getattr(uvtemp, uvcal_read_method)(uvlist[0], use_future_array_shapes=True) + getattr(uvtemp, uvcal_read_method)(uvlist[0]) nintegrations = len(uvlist) * uvtemp.Ntimes # Determine the actual files to store # We will drop kt_size / (integrations per file) files at the start and @@ -2641,12 +2600,8 @@ def xrfi_h3c_idr2_1_run(ocalfits_files, acalfits_files, model_files, data_files, # Initial run on cal data products # Calculate metric on abscal data uvc_a = UVCal() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - getattr(uvc_a, uvcal_read_method)(acalfits_files, use_future_array_shapes=True) - uvf_apriori = UVFlag( - uvc_a, mode='flag', copy_flags=True, label='A priori flags.', use_future_array_shapes=True - ) + getattr(uvc_a, uvcal_read_method)(acalfits_files) + uvf_apriori = UVFlag(uvc_a, mode='flag', copy_flags=True, label='A priori flags.') uvf_ag, uvf_agf = xrfi_pipe(uvc_a, alg='detrend_medfilt', Kt=kt_size, Kf=kf_size, xants=xants, cal_mode='gain', sig_init=sig_init, sig_adj=sig_adj, label='Abscal gains, round 1.', **check_kwargs) @@ -2656,9 +2611,7 @@ def xrfi_h3c_idr2_1_run(ocalfits_files, acalfits_files, model_files, data_files, # Calculate metric on omnical data uvc_o = UVCal() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - getattr(uvc_o, uvcal_read_method)(ocalfits_files, use_future_array_shapes=True) + getattr(uvc_o, uvcal_read_method)(ocalfits_files) flag_apply(uvf_apriori, uvc_o, keep_existing=True, run_check=run_check, check_extra=check_extra, run_check_acceptability=run_check_acceptability) @@ -2670,9 +2623,7 @@ def xrfi_h3c_idr2_1_run(ocalfits_files, acalfits_files, model_files, data_files, label='Omnical chisq, round 1.', **check_kwargs) # Calculate metric on model vis - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uv_v = UVData.from_file(model_files, axis='blt', use_future_array_shapes=True) + uv_v = UVData.from_file(model_files, axis='blt') uvf_v, uvf_vf = xrfi_pipe(uv_v, alg='detrend_medfilt', xants=[], Kt=kt_size, Kf=kf_size, sig_init=sig_init, sig_adj=sig_adj, label='Omnical visibility solutions, round 1.', @@ -2710,9 +2661,7 @@ def xrfi_h3c_idr2_1_run(ocalfits_files, acalfits_files, model_files, data_files, # Second round -- use init flags to mask and recalculate everything # Read in data file uv_d = UVData() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uv_d = UVData.from_file(data_files, axis='blt', use_future_array_shapes=True) + uv_d = UVData.from_file(data_files, axis='blt') for uv in [uvc_o, uvc_a, uv_v, uv_d]: flag_apply(uvf_init, uv, keep_existing=True, force_pol=True, **check_kwargs) @@ -2798,9 +2747,7 @@ def xrfi_h3c_idr2_1_run(ocalfits_files, acalfits_files, model_files, data_files, # output files for those, but flag everything. # Read metadata from first file to get integrations per file. - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvtemp = UVData.from_file(data_files[0], read_data=False, use_future_array_shapes=True) + uvtemp = UVData.from_file(data_files[0], read_data=False) nintegrations = len(data_files) * uvtemp.Ntimes # Calculate number of files to drop on edges, rounding up. ndrop = int(np.ceil(kt_size / uvtemp.Ntimes)) @@ -2915,25 +2862,19 @@ def day_threshold_run(data_files, history, nsig_f=7., nsig_t=7., if np.all([len(f) > 0 for f in files1_all]) and np.all([len(f) > 0 for f in files2_all]): files1 = [glob.glob(d + '/*' + ext + '1.h5')[0] for d in xrfi_dirs] files2 = [glob.glob(d + '/*' + ext + '2.h5')[0] for d in xrfi_dirs] - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf1 = UVFlag(files1, use_future_array_shapes=True) - uvf2 = UVFlag(files2, use_future_array_shapes=True) + uvf1 = UVFlag(files1) + uvf2 = UVFlag(files2) uvf2.metric_array = np.where(np.isinf(uvf2.metric_array), uvf1.metric_array, uvf2.metric_array) filled_metrics.append(uvf2) elif np.all([len(f) > 0 for f in files2_all]): # some flags only exist in round2 (data for example). files = [glob.glob(d + '/*' + ext + '2.h5')[0] for d in xrfi_dirs] - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - filled_metrics.append(UVFlag(files, use_future_array_shapes=True)) + filled_metrics.append(UVFlag(files)) elif np.all([len(f) > 0 for f in files1_all]): # some flags only exist in round1 (if we chose median filtering only for example). files = [glob.glob(d + '/*' + ext + '1.h5')[0] for d in xrfi_dirs] - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - filled_metrics.append(UVFlag(files, use_future_array_shapes=True)) + filled_metrics.append(UVFlag(files)) else: filled_metrics.append(None) filled_metrics_that_exist = [f for f in filled_metrics if f is not None] @@ -2960,9 +2901,7 @@ def day_threshold_run(data_files, history, nsig_f=7., nsig_t=7., try: ext_here = f'{mext.replace("metrics", "flags")}{rnd}.h5' files = [glob.glob(f'{d}/*.{ext_here}')[0] for d in xrfi_dirs] - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvf_here = UVFlag(files, use_future_array_shapes=True) + uvf_here = UVFlag(files) uvf_total |= uvf_here except IndexError: pass @@ -2985,9 +2924,7 @@ def day_threshold_run(data_files, history, nsig_f=7., nsig_t=7., abs_in = '.'.join([basename, incal_ext, 'calfits']) abs_out = '.'.join([basename, outcal_ext, 'calfits']) # abscal flagging only happens if the abscal files exist. - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - getattr(uvc_a, uvcal_read_method)(abs_in, use_future_array_shapes=True) + getattr(uvc_a, uvcal_read_method)(abs_in) # select the times from the file we are going to flag uvf_file = uvf_total.select(times=uvc_a.time_array, inplace=False) @@ -3113,9 +3050,7 @@ def xrfi_h1c_run(indata, history, infile_format='miriad', extension='flags.h5', filename = indata elif not isinstance(filename, str): raise ValueError('filename must be string path to file.') - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvd = UVData.from_file(filename, use_future_array_shapes=True) + uvd = UVData.from_file(filename) # append to history history = 'Flagging command: "' + history + '", Using ' + __version__ @@ -3152,9 +3087,7 @@ def xrfi_h1c_run(indata, history, infile_format='miriad', extension='flags.h5', # Flag on model visibilities if model_file is not None: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvm = UVData.from_file(model_file, use_future_array_shapes=True) + uvm = UVData.from_file(model_file) if indata is not None and not ( np.allclose(np.unique(uvd.time_array), np.unique(uvm.time_array), atol=1e-5, rtol=0 ) and np.allclose( @@ -3180,9 +3113,7 @@ def xrfi_h1c_run(indata, history, infile_format='miriad', extension='flags.h5', # Flag on gain solutions and chisquared values if calfits_file is not None: uvc = UVCal() - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - getattr(uvc, uvcal_read_method)(calfits_file, use_future_array_shapes=True) + getattr(uvc, uvcal_read_method)(calfits_file) if indata is not None and not ( np.allclose(np.unique(uvd.time_array), np.unique(uvc.time_array), atol=1e-5, rtol=0 ) and np.allclose( @@ -3286,9 +3217,7 @@ def xrfi_h1c_apply(filename, history, infile_format='miriad', xrfi_path='', raise AssertionError('xrfi_apply currently only takes a single data file.') if isinstance(filename, (list, np.ndarray, tuple)): filename = filename[0] - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", "Future array shapes are now always used") - uvd = UVData.from_file(filename, use_future_array_shapes=True) + uvd = UVData.from_file(filename) full_list = [] # Read in flag file diff --git a/pyproject.toml b/pyproject.toml index c767bf48..d310a226 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "astropy>=5.0.4", "h5py>=3.1", "pyyaml", - "pyuvdata>=2.3", + "pyuvdata>=3.0", "hera_filters" ] classifiers = [ From d99df92ee5e3fbcf36af54aff3be47a042c4a586 Mon Sep 17 00:00:00 2001 From: Steven Murray Date: Tue, 9 Jul 2024 03:07:33 +0200 Subject: [PATCH 2/4] style: apply pre-commit --- .coveragerc | 4 ++-- .gitignore | 2 +- .pre-commit-config.yaml | 2 ++ hera_qm/metrics_io.py | 6 +++--- hera_qm/omnical_metrics.py | 1 - hera_qm/tests/test_ant_metrics.py | 4 ++-- hera_qm/tests/test_auto_metrics.py | 2 +- hera_qm/tests/test_xrfi.py | 2 +- 8 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.coveragerc b/.coveragerc index 526c407d..7d68ad0a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,9 +3,9 @@ omit = */tests/* */data/* */scripts/* - + [report] omit = */tests/* */data/* - */scripts/* \ No newline at end of file + */scripts/* diff --git a/.gitignore b/.gitignore index 871c0ee2..99cc61ae 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,4 @@ hera_qm/data/test_output/ant_metrics_output.json hera_qm/_version.py artifacts/* -result.md \ No newline at end of file +result.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 53669e6e..46081c55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +exclude: '(^docs/conf.py|^hera_qm/data/|.codecov.yml)' + ci: autoupdate_schedule: monthly diff --git a/hera_qm/metrics_io.py b/hera_qm/metrics_io.py index e7f17f70..3887b827 100644 --- a/hera_qm/metrics_io.py +++ b/hera_qm/metrics_io.py @@ -117,11 +117,11 @@ def _recursively_save_dict_to_group(h5file, path, in_dict): """ allowed_types = (np.ndarray, np.float32, float, int, bytes, str, list, bool, np.bool_) - - + + compressable_types = (np.ndarray, list) for key in in_dict: - # Catch case with numpy 2 where keys of type (np.int32(ant), 'polstr')) are + # Catch case with numpy 2 where keys of type (np.int32(ant), 'polstr')) are # converted to string with the full np.int32 identifier in them. if isinstance(key, tuple): key = tuple(int(k) if isinstance(k, (int, np.int32, np.int64)) else k for k in key) diff --git a/hera_qm/omnical_metrics.py b/hera_qm/omnical_metrics.py index ced1dc9a..67e6029a 100644 --- a/hera_qm/omnical_metrics.py +++ b/hera_qm/omnical_metrics.py @@ -12,7 +12,6 @@ import json import copy import os -import warnings def get_omnical_metrics_dict(): diff --git a/hera_qm/tests/test_ant_metrics.py b/hera_qm/tests/test_ant_metrics.py index ac90cda4..16f5328c 100644 --- a/hera_qm/tests/test_ant_metrics.py +++ b/hera_qm/tests/test_ant_metrics.py @@ -364,11 +364,11 @@ def test_ant_metrics_run_and_load_antenna_metrics(): # test a priori flagging via YAML four_pol_uvh5 = DATA_PATH + '/zen.2457698.40355.full_pol_test.uvh5' apf_yaml = os.path.join(DATA_PATH, 'a_priori_flags_sample.yaml') - + # SGM: not sure if the following warning *should* be raised. with pytest.warns(RuntimeWarning, match='All-NaN slice encountered'): am = ant_metrics.ant_metrics_run(four_pol_uvh5, diff_files=four_pol_uvh5, overwrite=True, a_priori_xants_yaml=apf_yaml, verbose=True) - + am_hdf5 = ant_metrics.load_antenna_metrics(four_pol_uvh5.replace('.uvh5', '.ant_metrics.hdf5')) for ant in [(0, 'Jee'), (0, 'Jnn'), (10, 'Jee'), (10, 'Jnn'), (1, 'Jee'), (3, 'Jnn')]: assert ant in am_hdf5['xants'] diff --git a/hera_qm/tests/test_auto_metrics.py b/hera_qm/tests/test_auto_metrics.py index e0406a2f..ce88e238 100644 --- a/hera_qm/tests/test_auto_metrics.py +++ b/hera_qm/tests/test_auto_metrics.py @@ -221,7 +221,7 @@ def test_auto_metrics_run(): warnings.filterwarnings("ignore", message="K1 value 8 is larger than the data of dimension 4; using the size of the data for the kernel size") warnings.filterwarnings("ignore", message="Mean of empty slice") warnings.filterwarnings("ignore", message="Degrees of freedom <= 0 for slice") - + ex_ants, modzs, spectra, flags = auto_metrics.auto_metrics_run(metrics_outfile, auto_files, median_round_modz_cut=75., mean_round_modz_cut=5., edge_cut=100, Kt=8, Kf=8, sig_init=5.0, sig_adj=2.0, diff --git a/hera_qm/tests/test_xrfi.py b/hera_qm/tests/test_xrfi.py index cd4599c2..234f221e 100644 --- a/hera_qm/tests/test_xrfi.py +++ b/hera_qm/tests/test_xrfi.py @@ -1398,7 +1398,7 @@ def test_xrfi_run_step(tmpdir): assert np.all(np.isclose(uvf_a1.flag_array, uvf_a2.flag_array)) assert np.all(np.isclose(uvf1.metric_array, uvf2.metric_array, atol=1e-5, rtol=1e-5)) # hit one line involving uvcal reinitialization and include yaml file. - + with pytest.warns(RuntimeWarning, match="invalid value encountered in subtract"): xrfi.xrfi_run_step(uv_files=ocal_file, calculate_uvf_apriori=True, run_filter=True, reinitialize=True, a_priori_flag_yaml=a_priori_flag_integrations) # test invalid data type error From 13b9df295fa0a942521465ea5a25d9394af6f581 Mon Sep 17 00:00:00 2001 From: Steven Murray Date: Tue, 9 Jul 2024 13:11:02 +0200 Subject: [PATCH 3/4] maint: deprecated py39 --- .github/workflows/ci.yml | 2 +- hera_qm/tests/test_vis_metrics.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81e1b1d1..84b9bd9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.9, '3.10', "3.11", "3.12"] + python-version: ['3.10', "3.11", "3.12"] fail-fast: false steps: diff --git a/hera_qm/tests/test_vis_metrics.py b/hera_qm/tests/test_vis_metrics.py index 05e3705e..3e64ba05 100644 --- a/hera_qm/tests/test_vis_metrics.py +++ b/hera_qm/tests/test_vis_metrics.py @@ -10,13 +10,13 @@ import copy import pytest import warnings -from erfa.core import ErfaWarning pytestmark = pytest.mark.filterwarnings( "ignore:The uvw_array does not match the expected values given the antenna positions.", ) @pytest.fixture(scope='function') +@pytest.mark.filterwarnings("ignore:ERFA function") def vismetrics_data(): data = UVData() filename = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA') @@ -29,8 +29,7 @@ def vismetrics_data(): while data.Ntimes < 90: d2 = copy.deepcopy(data) d2.time_array += d2.time_array.max() + d2.integration_time / (24 * 3600) - with pytest.raises(ErfaWarning, match='ERFA function'): - data += d2 + data += d2 ntimes = data.Ntimes nchan = data.Nfreqs data1 = qmtest.noise(size=(ntimes, nchan)) From 871358f59e96dfeb05ae2433c669f0647ecd640c Mon Sep 17 00:00:00 2001 From: Steven Murray Date: Tue, 9 Jul 2024 13:34:18 +0200 Subject: [PATCH 4/4] test: catch erfa warnings if emitted? --- hera_qm/tests/test_vis_metrics.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/hera_qm/tests/test_vis_metrics.py b/hera_qm/tests/test_vis_metrics.py index 3e64ba05..3f36d9a1 100644 --- a/hera_qm/tests/test_vis_metrics.py +++ b/hera_qm/tests/test_vis_metrics.py @@ -10,13 +10,13 @@ import copy import pytest import warnings +from erfa.core import ErfaWarning pytestmark = pytest.mark.filterwarnings( "ignore:The uvw_array does not match the expected values given the antenna positions.", ) @pytest.fixture(scope='function') -@pytest.mark.filterwarnings("ignore:ERFA function") def vismetrics_data(): data = UVData() filename = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA') @@ -26,10 +26,12 @@ def vismetrics_data(): data.select(antenna_nums=data.get_ants()[0:10]) data.select(freq_chans=range(100)) # Data file only has three times... need more. - while data.Ntimes < 90: - d2 = copy.deepcopy(data) - d2.time_array += d2.time_array.max() + d2.integration_time / (24 * 3600) - data += d2 + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=ErfaWarning) + while data.Ntimes < 90: + d2 = copy.deepcopy(data) + d2.time_array += d2.time_array.max() + d2.integration_time / (24 * 3600) + data += d2 ntimes = data.Ntimes nchan = data.Nfreqs data1 = qmtest.noise(size=(ntimes, nchan))