diff --git a/.travis.yml b/.travis.yml index 65fc1b39..fe5ebd70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.6" - "3.7" - "3.8" + env: global: - COVERALLS_PARALLEL=true diff --git a/hera_qm/ant_metrics.py b/hera_qm/ant_metrics.py index 974f4b85..fef1f6f6 100644 --- a/hera_qm/ant_metrics.py +++ b/hera_qm/ant_metrics.py @@ -809,8 +809,8 @@ def _run_all_metrics(self, run_mean_vij=True, run_red_corr=True, if run_red_corr: metNames.append('redCorr') - - redCorr = self.red_corr_metrics(pols=['xx', 'yy'], + pols = [pol for pol in self.pols if pol[0] == pol[1]] + redCorr = self.red_corr_metrics(pols=pols, xants=self.xants, rawMetric=True) metVals.append(redCorr) diff --git a/hera_qm/data/example_two_polarization_firstcal_results.hdf5 b/hera_qm/data/example_two_polarization_firstcal_results.hdf5 index c748f17f..628dd8e1 100644 Binary files a/hera_qm/data/example_two_polarization_firstcal_results.hdf5 and b/hera_qm/data/example_two_polarization_firstcal_results.hdf5 differ diff --git a/hera_qm/firstcal_metrics.py b/hera_qm/firstcal_metrics.py index c3fafd7d..0bfe6227 100644 --- a/hera_qm/firstcal_metrics.py +++ b/hera_qm/firstcal_metrics.py @@ -348,7 +348,8 @@ def __init__(self, calfits_files, use_gp=True): self.UVC = UVCal() self.UVC.read_calfits(calfits_files) - self.pols = np.array([uvutils.polnum2str(jones) for jones in self.UVC.jones_array]) + self.pols = np.array([uvutils.polnum2str(jones, x_orientation=self.UVC.x_orientation) + for jones in self.UVC.jones_array]) self.Npols = self.pols.size # get file prefix diff --git a/hera_qm/metrics_io.py b/hera_qm/metrics_io.py index 837dda9e..2a477ede 100644 --- a/hera_qm/metrics_io.py +++ b/hera_qm/metrics_io.py @@ -293,6 +293,18 @@ def write_metric_file(filename, input_dict, overwrite=False): # Create group for metrics data in file mgrp = outfile.create_group('Metrics') + mgrp.attrs['group_is_ordered'] = False + if isinstance(input_dict, OrderedDict): + mgrp.attrs['group_is_ordered'] = True + + # Generate additional dataset in an ordered group to save + # the order of the group + key_order = np.array(list(input_dict.keys())).astype('S') + key_set = mgrp.create_dataset('key_order', data=key_order) + + # Add boolean attribute to determine if key is a string + # Used to parse keys saved to dicts when reading + key_set.attrs['key_is_string'] = True mgrp.attrs['key_is_string'] = True _recursively_save_dict_to_group(outfile, "/Metrics/", input_dict) @@ -379,7 +391,7 @@ def _parse_key(key): return int(str(key)) except ValueError: # is key an antpol tuple? - antpol_regex = r"(\([0-9]*?, \'[xy]\'\))" + antpol_regex = r"(\([0-9]*?, \'[xyne]\'\))" matches = re.findall(antpol_regex, key) try: # split tuple into antenna number and polarization @@ -541,7 +553,7 @@ def _parse_dict(input_str, value_type=int): """ # use regex to extract keys and values # assumes keys are antpols and values are numbers - dict_regex = r'(\([0-9]*?, \'[xy]\'\)): (.*?)[,}]' + dict_regex = r'(\([0-9]*?, \'[xyne]\'\)): (.*?)[,}]' key_vals = re.findall(dict_regex, input_str) # initialize output @@ -606,7 +618,7 @@ def _parse_list_of_antpols(input_str): """ # use regex to extract entries # assumes list entries are antpols - list_regex = r"(\([0-9]*?, \'[xy]\'\))" + list_regex = r"(\([0-9]*?, \'[xyne]\'\))" entries = re.findall(list_regex, input_str) # initialize output @@ -837,11 +849,13 @@ def _recursively_validate_dict(in_dict): in_dict[key] = _reds_dict_to_list(in_dict[key]) if key in antpair_keys and key != 'reds': - in_dict[key] = list(map(tuple, in_dict[key])) + in_dict[key] = [tuple((int(a1), int(a2))) + for pair in in_dict[key] + for a1, a2 in pair] if key in antpol_keys: - in_dict[key] = [tuple((ant, qm_utils._bytes_to_str(pol))) - if isinstance(pol, bytes) else tuple((ant, (pol))) + in_dict[key] = [tuple((int(ant), qm_utils._bytes_to_str(pol))) + if isinstance(pol, bytes) else tuple((int(ant), (pol))) for ant, pol in in_dict[key]] if key in known_string_keys and isinstance(in_dict[key], bytes): @@ -851,7 +865,7 @@ def _recursively_validate_dict(in_dict): in_dict[key] = [qm_utils._bytes_to_str(n) if isinstance(n, bytes) else n for n in in_dict[key]] - if isinstance(in_dict[key], np.int64): + if isinstance(in_dict[key], (np.int64, np.int32)): in_dict[key] = np.int(in_dict[key]) if isinstance(in_dict[key], dict): @@ -890,7 +904,19 @@ def load_metric_file(filename): else: with h5py.File(filename, 'r') as infile: metric_dict = _recursively_load_dict_to_group(infile, "/Header/") - metric_dict.update(_recursively_load_dict_to_group(infile, "/Metrics/")) + metric_item = infile["/Metrics/"] + if hasattr(metric_item, 'attrs'): + if 'group_is_ordered' in metric_item.attrs: + group_is_ordered = metric_item.attrs["group_is_ordered"] + else: + group_is_ordered = False + else: + group_is_ordered = False + metric_dict.update( + _recursively_load_dict_to_group(infile, "/Metrics/", + group_is_ordered=group_is_ordered + ) + ) _recursively_validate_dict(metric_dict) return metric_dict diff --git a/hera_qm/tests/test_firstcal_metrics.py b/hera_qm/tests/test_firstcal_metrics.py index 0b918832..fe35a336 100644 --- a/hera_qm/tests/test_firstcal_metrics.py +++ b/hera_qm/tests/test_firstcal_metrics.py @@ -45,27 +45,27 @@ def test_init(firstcal_setup): def test_run_metrics(firstcal_setup): firstcal_setup.FC.run_metrics(std_cut=1.0) - assert firstcal_setup.FC.metrics['yy']['good_sol'] is True - assert firstcal_setup.FC.metrics['yy']['bad_ants'] == [] - assert 9 in firstcal_setup.FC.metrics['yy']['z_scores'] - assert 9 in firstcal_setup.FC.metrics['yy']['ant_std'] - assert 9 in firstcal_setup.FC.metrics['yy']['ant_avg'] - assert 9 in firstcal_setup.FC.metrics['yy']['ants'] - assert 9 in firstcal_setup.FC.metrics['yy']['z_scores'] - assert 9 in firstcal_setup.FC.metrics['yy']['ant_z_scores'] - assert np.isclose(1.0, firstcal_setup.FC.metrics['yy']['std_cut']) - assert np.isclose(firstcal_setup.FC.metrics['yy']['agg_std'], 0.044662349588061437) - assert np.isclose(firstcal_setup.FC.metrics['yy']['max_std'], 0.089829821120782846) - assert 'yy' == firstcal_setup.FC.metrics['yy']['pol'] + assert firstcal_setup.FC.metrics['nn']['good_sol'] is True + assert firstcal_setup.FC.metrics['nn']['bad_ants'] == [] + assert 9 in firstcal_setup.FC.metrics['nn']['z_scores'] + assert 9 in firstcal_setup.FC.metrics['nn']['ant_std'] + assert 9 in firstcal_setup.FC.metrics['nn']['ant_avg'] + assert 9 in firstcal_setup.FC.metrics['nn']['ants'] + assert 9 in firstcal_setup.FC.metrics['nn']['z_scores'] + assert 9 in firstcal_setup.FC.metrics['nn']['ant_z_scores'] + assert np.isclose(1.0, firstcal_setup.FC.metrics['nn']['std_cut']) + assert np.isclose(firstcal_setup.FC.metrics['nn']['agg_std'], 0.044662349588061437) + assert np.isclose(firstcal_setup.FC.metrics['nn']['max_std'], 0.089829821120782846) + assert 'nn' == firstcal_setup.FC.metrics['nn']['pol'] # Test bad ants detection firstcal_setup.FC.delay_fluctuations[0, :] *= 1000 firstcal_setup.FC.run_metrics() - assert firstcal_setup.FC.ants[0] == firstcal_setup.FC.metrics['yy']['bad_ants'] + assert firstcal_setup.FC.ants[0] == firstcal_setup.FC.metrics['nn']['bad_ants'] # Test bad full solution firstcal_setup.FC.delay_fluctuations[1:, :] *= 1000 firstcal_setup.FC.run_metrics() - assert firstcal_setup.FC.metrics['yy']['good_sol'] is False + assert firstcal_setup.FC.metrics['nn']['good_sol'] is False def test_write_error_bad_type(firstcal_setup): @@ -217,9 +217,9 @@ def test_rotated_metrics(): FC = firstcal_metrics.FirstCalMetrics(infile) FC.run_metrics(std_cut=0.5) # test pickup of rotant key - assert 'rot_ants' in FC.metrics['xx'].keys() + assert 'rot_ants' in FC.metrics['ee'].keys() # test rotants is correct - assert [43] == FC.metrics['xx']['rot_ants'] + assert [43] == FC.metrics['ee']['rot_ants'] def test_delay_smoothing(): diff --git a/hera_qm/utils.py b/hera_qm/utils.py index 476dab49..4e4ce2d9 100644 --- a/hera_qm/utils.py +++ b/hera_qm/utils.py @@ -417,7 +417,8 @@ def generate_fullpol_file_list(files, pol_list): # convert the polarization array to strings and compare with the # expected input. # If anyone file is not a full-pol file then this will be false. - input_pols = uvutils.polnum2str(uvd.polarization_array) + input_pols = uvutils.polnum2str(uvd.polarization_array, + x_orientation=uvd.x_orientation) full_pol_check = np.array_equal(np.sort(input_pols), np.sort(pol_list)) if not full_pol_check: