Merge pull request #310 from HERA-Team/x_orientation_update

X orientation update

.travis.yml

@@ -5,6 +5,7 @@ python:
   - "3.6"
   - "3.7"
   - "3.8"
+
 env:
   global:
     - COVERALLS_PARALLEL=true

hera_qm/ant_metrics.py

@@ -809,8 +809,8 @@ def _run_all_metrics(self, run_mean_vij=True, run_red_corr=True,
 
         if run_red_corr:
             metNames.append('redCorr')
-
-            redCorr = self.red_corr_metrics(pols=['xx', 'yy'],
+            pols = [pol for pol in self.pols if pol[0] == pol[1]]
+            redCorr = self.red_corr_metrics(pols=pols,
                                             xants=self.xants,
                                             rawMetric=True)
             metVals.append(redCorr)

hera_qm/firstcal_metrics.py

@@ -348,7 +348,8 @@ def __init__(self, calfits_files, use_gp=True):
         self.UVC = UVCal()
         self.UVC.read_calfits(calfits_files)
 
-        self.pols = np.array([uvutils.polnum2str(jones) for jones in self.UVC.jones_array])
+        self.pols = np.array([uvutils.polnum2str(jones, x_orientation=self.UVC.x_orientation)
+                              for jones in self.UVC.jones_array])
         self.Npols = self.pols.size
 
         # get file prefix

hera_qm/metrics_io.py

@@ -293,6 +293,18 @@ def write_metric_file(filename, input_dict, overwrite=False):
 
             # Create group for metrics data in file
             mgrp = outfile.create_group('Metrics')
+            mgrp.attrs['group_is_ordered'] = False
+            if isinstance(input_dict, OrderedDict):
+                mgrp.attrs['group_is_ordered'] = True
+
+                # Generate additional dataset in an ordered group to save
+                # the order of the group
+                key_order = np.array(list(input_dict.keys())).astype('S')
+                key_set = mgrp.create_dataset('key_order', data=key_order)
+
+                # Add boolean attribute to determine if key is a string
+                # Used to parse keys saved to dicts when reading
+                key_set.attrs['key_is_string'] = True
             mgrp.attrs['key_is_string'] = True
             _recursively_save_dict_to_group(outfile, "/Metrics/", input_dict)
 
@@ -379,7 +391,7 @@ def _parse_key(key):
         return int(str(key))
     except ValueError:
         # is key an antpol tuple?
-        antpol_regex = r"(\([0-9]*?, \'[xy]\'\))"
+        antpol_regex = r"(\([0-9]*?, \'[xyne]\'\))"
         matches = re.findall(antpol_regex, key)
         try:
             # split tuple into antenna number and polarization
@@ -541,7 +553,7 @@ def _parse_dict(input_str, value_type=int):
     """
     # use regex to extract keys and values
     # assumes keys are antpols and values are numbers
-    dict_regex = r'(\([0-9]*?, \'[xy]\'\)): (.*?)[,}]'
+    dict_regex = r'(\([0-9]*?, \'[xyne]\'\)): (.*?)[,}]'
     key_vals = re.findall(dict_regex, input_str)
 
     # initialize output
@@ -606,7 +618,7 @@ def _parse_list_of_antpols(input_str):
     """
     # use regex to extract entries
     # assumes list entries are antpols
-    list_regex = r"(\([0-9]*?, \'[xy]\'\))"
+    list_regex = r"(\([0-9]*?, \'[xyne]\'\))"
     entries = re.findall(list_regex, input_str)
 
     # initialize output
@@ -837,11 +849,13 @@ def _recursively_validate_dict(in_dict):
             in_dict[key] = _reds_dict_to_list(in_dict[key])
 
         if key in antpair_keys and key != 'reds':
-            in_dict[key] = list(map(tuple, in_dict[key]))
+            in_dict[key] = [tuple((int(a1), int(a2)))
+                            for pair in in_dict[key]
+                            for a1, a2 in pair]
 
         if key in antpol_keys:
-            in_dict[key] = [tuple((ant, qm_utils._bytes_to_str(pol)))
-                            if isinstance(pol, bytes) else tuple((ant, (pol)))
+            in_dict[key] = [tuple((int(ant), qm_utils._bytes_to_str(pol)))
+                            if isinstance(pol, bytes) else tuple((int(ant), (pol)))
                             for ant, pol in in_dict[key]]
 
         if key in known_string_keys and isinstance(in_dict[key], bytes):
@@ -851,7 +865,7 @@ def _recursively_validate_dict(in_dict):
             in_dict[key] = [qm_utils._bytes_to_str(n) if isinstance(n, bytes)
                             else n for n in in_dict[key]]
 
-        if isinstance(in_dict[key], np.int64):
+        if isinstance(in_dict[key], (np.int64, np.int32)):
             in_dict[key] = np.int(in_dict[key])
 
         if isinstance(in_dict[key], dict):
@@ -890,7 +904,19 @@ def load_metric_file(filename):
     else:
         with h5py.File(filename, 'r') as infile:
             metric_dict = _recursively_load_dict_to_group(infile, "/Header/")
-            metric_dict.update(_recursively_load_dict_to_group(infile, "/Metrics/"))
+            metric_item = infile["/Metrics/"]
+            if hasattr(metric_item, 'attrs'):
+                if 'group_is_ordered' in metric_item.attrs:
+                    group_is_ordered = metric_item.attrs["group_is_ordered"]
+                else:
+                    group_is_ordered = False
+            else:
+                group_is_ordered = False
+            metric_dict.update(
+                _recursively_load_dict_to_group(infile, "/Metrics/",
+                                                group_is_ordered=group_is_ordered
+                                                )
+            )
 
     _recursively_validate_dict(metric_dict)
     return metric_dict

hera_qm/tests/test_firstcal_metrics.py

@@ -45,27 +45,27 @@ def test_init(firstcal_setup):
 
 def test_run_metrics(firstcal_setup):
     firstcal_setup.FC.run_metrics(std_cut=1.0)
-    assert firstcal_setup.FC.metrics['yy']['good_sol'] is True
-    assert firstcal_setup.FC.metrics['yy']['bad_ants'] == []
-    assert 9 in firstcal_setup.FC.metrics['yy']['z_scores']
-    assert 9 in firstcal_setup.FC.metrics['yy']['ant_std']
-    assert 9 in firstcal_setup.FC.metrics['yy']['ant_avg']
-    assert 9 in firstcal_setup.FC.metrics['yy']['ants']
-    assert 9 in firstcal_setup.FC.metrics['yy']['z_scores']
-    assert 9 in firstcal_setup.FC.metrics['yy']['ant_z_scores']
-    assert np.isclose(1.0, firstcal_setup.FC.metrics['yy']['std_cut'])
-    assert np.isclose(firstcal_setup.FC.metrics['yy']['agg_std'], 0.044662349588061437)
-    assert np.isclose(firstcal_setup.FC.metrics['yy']['max_std'], 0.089829821120782846)
-    assert 'yy' == firstcal_setup.FC.metrics['yy']['pol']
+    assert firstcal_setup.FC.metrics['nn']['good_sol'] is True
+    assert firstcal_setup.FC.metrics['nn']['bad_ants'] == []
+    assert 9 in firstcal_setup.FC.metrics['nn']['z_scores']
+    assert 9 in firstcal_setup.FC.metrics['nn']['ant_std']
+    assert 9 in firstcal_setup.FC.metrics['nn']['ant_avg']
+    assert 9 in firstcal_setup.FC.metrics['nn']['ants']
+    assert 9 in firstcal_setup.FC.metrics['nn']['z_scores']
+    assert 9 in firstcal_setup.FC.metrics['nn']['ant_z_scores']
+    assert np.isclose(1.0, firstcal_setup.FC.metrics['nn']['std_cut'])
+    assert np.isclose(firstcal_setup.FC.metrics['nn']['agg_std'], 0.044662349588061437)
+    assert np.isclose(firstcal_setup.FC.metrics['nn']['max_std'], 0.089829821120782846)
+    assert 'nn' == firstcal_setup.FC.metrics['nn']['pol']
 
     # Test bad ants detection
     firstcal_setup.FC.delay_fluctuations[0, :] *= 1000
     firstcal_setup.FC.run_metrics()
-    assert firstcal_setup.FC.ants[0] == firstcal_setup.FC.metrics['yy']['bad_ants']
+    assert firstcal_setup.FC.ants[0] == firstcal_setup.FC.metrics['nn']['bad_ants']
     # Test bad full solution
     firstcal_setup.FC.delay_fluctuations[1:, :] *= 1000
     firstcal_setup.FC.run_metrics()
-    assert firstcal_setup.FC.metrics['yy']['good_sol'] is False
+    assert firstcal_setup.FC.metrics['nn']['good_sol'] is False
 
 
 def test_write_error_bad_type(firstcal_setup):
@@ -217,9 +217,9 @@ def test_rotated_metrics():
     FC = firstcal_metrics.FirstCalMetrics(infile)
     FC.run_metrics(std_cut=0.5)
     # test pickup of rotant key
-    assert 'rot_ants' in FC.metrics['xx'].keys()
+    assert 'rot_ants' in FC.metrics['ee'].keys()
     # test rotants is correct
-    assert [43] == FC.metrics['xx']['rot_ants']
+    assert [43] == FC.metrics['ee']['rot_ants']
 
 
 def test_delay_smoothing():

hera_qm/utils.py

@@ -417,7 +417,8 @@ def generate_fullpol_file_list(files, pol_list):
         # convert the polarization array to strings and compare with the
         # expected input.
         # If anyone file is not a full-pol file then this will be false.
-        input_pols = uvutils.polnum2str(uvd.polarization_array)
+        input_pols = uvutils.polnum2str(uvd.polarization_array,
+                                        x_orientation=uvd.x_orientation)
         full_pol_check = np.array_equal(np.sort(input_pols), np.sort(pol_list))
 
         if not full_pol_check: