Merge pull request #464 from HERA-Team/use-pyuvdata3

Use pyuvdata3

.coveragerc

@@ -3,9 +3,9 @@ omit =
      */tests/*
      */data/*
      */scripts/*
-     
+
 [report]
 omit =
      */tests/*
      */data/*
-     */scripts/*
+     */scripts/*

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.9, '3.10', "3.11", "3.12"]
+        python-version: ['3.10', "3.11", "3.12"]
       fail-fast: false
 
     steps:

.gitignore

@@ -37,4 +37,4 @@ hera_qm/data/test_output/ant_metrics_output.json
 hera_qm/_version.py
 
 artifacts/*
-result.md
+result.md

.pre-commit-config.yaml

@@ -1,3 +1,5 @@
+exclude: '(^docs/conf.py|^hera_qm/data/|.codecov.yml)'
+
 ci:
   autoupdate_schedule: monthly
 

hera_qm/firstcal_metrics.py

@@ -345,10 +345,7 @@ def __init__(self, calfits_files, use_gp=True):
         # Instantiate UVCal and read calfits
         self.UVC = UVCal()
         with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore", "Future array shapes are now always used"
-            )
-            self.UVC.read_calfits(calfits_files, use_future_array_shapes=True)
+            self.UVC.read_calfits(calfits_files)
 
         if hasattr(self.UVC, "telescope"):
             x_orientation = self.UVC.telescope.x_orientation

hera_qm/metrics_io.py

@@ -117,8 +117,15 @@ def _recursively_save_dict_to_group(h5file, path, in_dict):
     """
     allowed_types = (np.ndarray, np.float32, float, int,
                      bytes, str, list, bool, np.bool_)
+
+
     compressable_types = (np.ndarray, list)
     for key in in_dict:
+        # Catch case with numpy 2 where keys of type (np.int32(ant), 'polstr')) are
+        # converted to string with the full np.int32 identifier in them.
+        if isinstance(key, tuple):
+            key = tuple(int(k) if isinstance(k, (int, np.int32, np.int64)) else k for k in key)
+
         key_str = str(key)
         if key == 'reds':
             in_dict[key] = _reds_list_to_dict(in_dict[key])
@@ -346,17 +353,16 @@ def _recursively_load_dict_to_group(h5file, path, group_is_ordered=False):
 
         item = h5file[path][key]
 
-        if isinstance(item, h5py.Dataset):
-            if item.attrs['key_is_string']:
-                out_dict[str(key)] = item[()]
-            else:
-                out_key = _parse_key(key)
+        if isinstance(item, (h5py.Dataset, h5py.Group)):
+            out_key = str(key) if item.attrs['key_is_string'] else _parse_key(key)
+            #print(key, out_key, item.attrs['key_is_string'])
+            if isinstance(item, h5py.Dataset):
                 out_dict[out_key] = item[()]
 
-        elif isinstance(item, h5py.Group):
-            out_key = str(key) if item.attrs['key_is_string'] else _parse_key(key)
-            out_dict[out_key] = _recursively_load_dict_to_group(h5file, (path + key + '/'),
-                                                                group_is_ordered=item.attrs["group_is_ordered"])
+            elif isinstance(item, h5py.Group):
+                out_dict[out_key] = _recursively_load_dict_to_group(
+                    h5file, (path + key + '/'), group_is_ordered=item.attrs["group_is_ordered"]
+                )
         else:
             raise TypeError("The HDF5 path: {} is not associated with either "
                             "a dataset or group object. "

hera_qm/omnical_metrics.py

@@ -12,7 +12,6 @@
 import json
 import copy
 import os
-import warnings
 
 
 def get_omnical_metrics_dict():
@@ -174,11 +173,7 @@ def load_firstcal_gains(fc_file):
         The polarization of the .calfits file.
     """
     uvc = UVCal()
-    with warnings.catch_warnings():
-        warnings.filterwarnings(
-            "ignore", "Future array shapes are now always used"
-        )
-        uvc.read_calfits(fc_file, use_future_array_shapes=True)
+    uvc.read_calfits(fc_file)
     fc_gains = np.moveaxis(uvc.gain_array, 1, 2)[:, :, :, 0]
     d_nu = np.mean(uvc.freq_array[1:] - uvc.freq_array[:-1])
     d_phi = np.abs(np.mean(np.angle(fc_gains)[:, :, 1:] - np.angle(fc_gains)[:, :, :-1], axis=2))
@@ -453,11 +448,7 @@ def __init__(self, omni_calfits, history=''):
 
         # Instantiate Data Object
         self.uv = UVCal()
-        with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore", "Future array shapes are now always used"
-            )
-            self.uv.read_calfits(omni_calfits, use_future_array_shapes=True)
+        self.uv.read_calfits(omni_calfits)
 
         # Get relevant metadata
         self.Nants = self.uv.Nants_data

hera_qm/tests/test_ant_metrics.py

@@ -364,7 +364,11 @@ def test_ant_metrics_run_and_load_antenna_metrics():
     # test a priori flagging via YAML
     four_pol_uvh5 = DATA_PATH + '/zen.2457698.40355.full_pol_test.uvh5'
     apf_yaml = os.path.join(DATA_PATH, 'a_priori_flags_sample.yaml')
-    am = ant_metrics.ant_metrics_run(four_pol_uvh5, diff_files=four_pol_uvh5, overwrite=True, a_priori_xants_yaml=apf_yaml, verbose=True)
+
+    # SGM: not sure if the following warning *should* be raised.
+    with pytest.warns(RuntimeWarning, match='All-NaN slice encountered'):
+        am = ant_metrics.ant_metrics_run(four_pol_uvh5, diff_files=four_pol_uvh5, overwrite=True, a_priori_xants_yaml=apf_yaml, verbose=True)
+
     am_hdf5 = ant_metrics.load_antenna_metrics(four_pol_uvh5.replace('.uvh5', '.ant_metrics.hdf5'))
     for ant in [(0, 'Jee'), (0, 'Jnn'), (10, 'Jee'), (10, 'Jnn'), (1, 'Jee'), (3, 'Jnn')]:
         assert ant in am_hdf5['xants']

hera_qm/tests/test_auto_metrics.py

@@ -219,6 +219,9 @@ def test_auto_metrics_run():
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", message="All-NaN slice encountered")
         warnings.filterwarnings("ignore", message="K1 value 8 is larger than the data of dimension 4; using the size of the data for the kernel size")
+        warnings.filterwarnings("ignore", message="Mean of empty slice")
+        warnings.filterwarnings("ignore", message="Degrees of freedom <= 0 for slice")
+
         ex_ants, modzs, spectra, flags = auto_metrics.auto_metrics_run(metrics_outfile, auto_files,
                                                                    median_round_modz_cut=75., mean_round_modz_cut=5.,
                                                                    edge_cut=100, Kt=8, Kf=8, sig_init=5.0, sig_adj=2.0,

hera_qm/tests/test_utils.py

@@ -286,7 +286,7 @@ def test_apply_yaml_flags_uvdata(tmpdir, filein, flag_freqs, flag_times, flag_an
     integration_flags = [0, 1] # integrations from yaml file that should be flagged.
     ant_flags = [0, 10, [1, 'Jee'], [3, 'Jnn']]
     uvd = UVData()
-    uvd.read(test_d_file, use_future_array_shapes=True)
+    uvd.read(test_d_file)
     uvd = utils.apply_yaml_flags(uvd, test_flag, flag_freqs=flag_freqs, flag_times=flag_times,
                                 flag_ants=flag_ants, unflag_first=True)
     if 'no_integrations' not in test_flag:
@@ -368,7 +368,7 @@ def test_apply_yaml_flags_uvcal(filein):
     test_flag = os.path.join(DATA_PATH, filein)
     test_c_file = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA.omni.calfits')
     uvc = UVCal()
-    uvc.read_calfits(test_c_file, use_future_array_shapes=True)
+    uvc.read_calfits(test_c_file)
 
     uvc = utils.apply_yaml_flags(uvc, test_flag, unflag_first=True)
     freq_regions = [(0, 110e6), (150e6, 155e6), (190e6, 200e6)] # frequencies from yaml file.
@@ -420,13 +420,12 @@ def test_apply_yaml_flags_uvcal(filein):
                          throw_away_flagged_ants=True, ant_indices_only=False)
 
 
-@pytest.mark.filterwarnings("ignore:Future array shapes are now always used")
 def test_apply_yaml_flags_errors():
     test_flag_jds = os.path.join(DATA_PATH, 'a_priori_flags_jds.yaml')
     test_c_file = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA.omni.calfits')
     # check NotImplementedErrors
     uvc = UVCal()
-    uvc.read_calfits(test_c_file, use_future_array_shapes=True)
+    uvc.read_calfits(test_c_file)
     # check that setting uv to an object that is not a subclass of UVCal or UVData throws a NotImplementedError
     pytest.raises(NotImplementedError, utils.apply_yaml_flags, 'uvdata', test_flag_jds)
 

hera_qm/tests/test_vis_metrics.py

@@ -10,6 +10,7 @@
 import copy
 import pytest
 import warnings
+from erfa.core import ErfaWarning
 
 pytestmark = pytest.mark.filterwarnings(
     "ignore:The uvw_array does not match the expected values given the antenna positions.",
@@ -19,18 +20,18 @@
 def vismetrics_data():
     data = UVData()
     filename = os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA')
-    with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", "Future array shapes are now always used")
-        data.read_miriad(filename, use_future_array_shapes=True, check_autos=False)
+    data.read_miriad(filename, check_autos=False)
 
     # massage the object to make it work with check_noise_variance
     data.select(antenna_nums=data.get_ants()[0:10])
     data.select(freq_chans=range(100))
     # Data file only has three times... need more.
-    while data.Ntimes < 90:
-        d2 = copy.deepcopy(data)
-        d2.time_array += d2.time_array.max() + d2.integration_time / (24 * 3600)
-        data += d2
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=ErfaWarning)
+        while data.Ntimes < 90:
+            d2 = copy.deepcopy(data)
+            d2.time_array += d2.time_array.max() + d2.integration_time / (24 * 3600)
+            data += d2
     ntimes = data.Ntimes
     nchan = data.Nfreqs
     data1 = qmtest.noise(size=(ntimes, nchan))
@@ -79,10 +80,9 @@ def test_check_noise_variance_inttime_error(vismetrics_data):
 def uvd():
     uvd = UVData()
     with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", "Future array shapes are now always used")
         uvd.read_miriad(
             os.path.join(DATA_PATH, 'zen.2458002.47754.xx.HH.uvA'),
-            projected=False, use_future_array_shapes=True,
+            projected=False,
             check_autos=False
         )
     return uvd
@@ -155,7 +155,7 @@ def test_plot_bl_bl_scatter(uvd):
 @pytest.mark.filterwarnings("ignore:Future array shapes are now always used")
 def test_sequential_diff():
     uvd = UVData()
-    uvd.read_miriad(os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA'), use_future_array_shapes=True)
+    uvd.read_miriad(os.path.join(DATA_PATH, 'zen.2457698.40355.xx.HH.uvcAA'))
 
     # diff across time
     uvd_diff = vis_metrics.sequential_diff(uvd, axis=0, pad=False)