Filter HDUs before loading to SpectrumList

CHANGES.rst

@@ -114,6 +114,9 @@ Mosviz
 
 - R-grism 2D spectrum data are now loaded with the correct orientation. [#1619]
 
+- Fixed a bug to skip targets not included in NIRISS source catalog, improving
+  lod times [#1696]
+
 Specviz
 ^^^^^^^
 

jdaviz/configs/mosviz/plugins/parsers.py

@@ -755,29 +755,33 @@ def mos_niriss_parser(app, data_dir):
             print(f"Loading: {flabel} sources")
 
             with fits.open(fname, memmap=False) as temp:
-                # treat all HDUs missing SRCTYPE attribute as extended sources
-                # TODO: Remove this once valid SRCTYPE values are present in all headers
-                for hdu in temp:
-                    if ("SRCTYPE" in hdu.header
-                            and (hdu.header["SRCTYPE"] in ("POINT", "EXTENDED"))):
-                        pass
-                    else:
-                        hdu.header["SRCTYPE"] = "EXTENDED"
-
-                # read all HDUs with SpectrumList, then only keep those that
-                # correspond with sources in catalog
-                # (read() is slow... would also LOVE to do this in one step!)
-                specs = SpectrumList.read(temp, format="JWST x1d multi")
-                specs_cut = [sp for sp in specs if sp.meta['header']['SOURCEID']
-                             in cat_id_dict.keys()]
+                # Filter out HDUs we care about
+                source_ids_to_filter = cat_id_dict.keys()
+                filtered_hdul = fits.HDUList([hdu for hdu in temp if (
+                    (hdu.name in ('PRIMARY', 'ASDF')) or
+                    (hdu.header.get('SOURCEID', None) in source_ids_to_filter))])
+
+                # SRCTYPE is required for the specutils JWST x1d reader. The reader will
+                # force this to POINT if not set. Under known cases, this field will be set
+                # for NIRISS programs; if it's not, something's gone wrong. Catch this
+                # warning and reraise as an error to warn users.
+                try:
+                    with warnings.catch_warnings():
+                        warnings.filterwarnings("error",
+                                                category=UserWarning,
+                                                message=".*SRCTYPE is missing or UNKNOWN*")
+                        specs = SpectrumList.read(filtered_hdul, format="JWST x1d multi")
+                except UserWarning as e:
+                    raise KeyError(f"The SRCTYPE keyword in the header of file {fname}"
+                                   "is not populated (expected values: EXTENDED or POINT)") from e
 
                 filter_name = fits.getheader(fname, ext=0).get('PUPIL')
 
                 # Orientation denoted by "C", "R", or "C+R" for combined spectra
                 orientation = flabel.split()[-1]
 
                 # update 1D labels and standardize metadata for table viewer
-                for sp in specs_cut:
+                for sp in specs:
                     if (
                         sp.meta['header']['SPORDER'] == 1
                         and sp.meta['header']['EXTNAME'] == 'EXTRACT1D'

jdaviz/configs/mosviz/tests/test_data_loading.py

@@ -193,26 +193,6 @@ def test_nirspec_loader(mosviz_helper, tmpdir):
     assert dc_15.meta['SOURCEID'] == 2315
 
 
-# This is another version of test_niriss_parser in test_parsers.py
-@pytest.mark.remote_data
-def test_niriss_loader(mosviz_helper, tmpdir):
-
-    test_data = 'https://stsci.box.com/shared/static/l2azhcqd3tvzhybdlpx2j2qlutkaro3z.zip'
-    fn = download_file(test_data, cache=True, timeout=30)
-    with ZipFile(fn, 'r') as sample_data_zip:
-        sample_data_zip.extractall(tmpdir)
-
-    level3_path = (pathlib.Path(tmpdir) / 'NIRISS_for_parser_p0171')
-
-    data_dir = level3_path
-
-    mosviz_helper.load_data(directory=data_dir, instrument='niriss')
-
-    assert len(mosviz_helper.app.data_collection) == 80
-    assert mosviz_helper.app.data_collection[0].label == "Image canucs F150W"
-    assert mosviz_helper.app.data_collection[-1].label == "MOS Table"
-
-
 @pytest.mark.remote_data
 def test_nirspec_fallback(mosviz_helper, tmpdir):
     '''

jdaviz/configs/mosviz/tests/test_parsers.py

@@ -1,48 +1,91 @@
 from zipfile import ZipFile
-import pathlib
 
 import pytest
 from astropy.utils.data import download_file
 
 from jdaviz.utils import PRIHDR_KEY, COMMENTCARD_KEY
 
 
-# This is another version of test_niriss_loader in test_data_loading.py
 @pytest.mark.remote_data
-def test_niriss_parser(mosviz_helper, tmpdir):
+@pytest.mark.filterwarnings('ignore', match="'(MJy/sr)^2' did not parse as fits unit")
+def test_niriss_parser(mosviz_helper, tmp_path):
+    '''
+    Tests loading a NIRISS dataset
+    This data set is a shortened version of the ERS program GLASS (Program 1324)
+    provided by Camilla Pacifici. This is in-flight, "real" JWST data
 
-    test_data = 'https://stsci.box.com/shared/static/l2azhcqd3tvzhybdlpx2j2qlutkaro3z.zip'
-    fn = download_file(test_data, cache=True, timeout=30)
-    with ZipFile(fn, 'r') as sample_data_zip:
-        sample_data_zip.extractall(tmpdir)
+    The spectra are jw01324001001_15101_00001_nis
+    The direct image is jw01324-o001_t001_niriss_clear-f200w
+    Please see JWST naming conventions for the above
 
-    level3_path = (pathlib.Path(tmpdir) / 'NIRISS_for_parser_p0171')
+    The dataset was uploaded to box by Duy Nguyen
+    '''
 
-    data_dir = level3_path
+    # Download data
+    test_data = 'https://stsci.box.com/shared/static/cr14xijcg572dglacochctr1kblsr89a.zip'
+    fn = download_file(test_data, cache=True, timeout=30)
 
-    mosviz_helper.load_niriss_data(data_dir)
+    # Extract to a known, temporary folder
+    with ZipFile(fn, 'r') as sample_data_zip:
+        sample_data_zip.extractall(tmp_path)
 
-    assert len(mosviz_helper.app.data_collection) == 80
+    mosviz_helper.load_data(directory=tmp_path, instrument="niriss")
+    assert len(mosviz_helper.app.data_collection) == 10
 
+    # The image should be the first in the data collection
     dc_0 = mosviz_helper.app.data_collection[0]
-    assert dc_0.label == "Image canucs F150W"
+    assert dc_0.label == "Image jw01324-o001 F200W"
     assert PRIHDR_KEY not in dc_0.meta
     assert COMMENTCARD_KEY not in dc_0.meta
     assert dc_0.meta['bunit_data'] == 'MJy/sr'  # ASDF metadata
 
-    dc_1 = mosviz_helper.app.data_collection[1]
-    assert dc_1.label == 'F150W Source 1 spec2d C'
-    assert PRIHDR_KEY in dc_1.meta
-    assert COMMENTCARD_KEY in dc_1.meta
-    assert dc_1.meta['SOURCEID'] == 1
-
-    dc_40 = mosviz_helper.app.data_collection[40]
-    assert dc_40.label == 'F150W Source 1 spec1d C'
-    assert PRIHDR_KEY not in dc_40.meta
-    assert COMMENTCARD_KEY in dc_40.meta
-    assert 'header' not in dc_40.meta
-    assert dc_40.meta['FILTER'] == 'GR150C'
-
+    # The MOS Table should be last in the data collection
     dc_tab = mosviz_helper.app.data_collection[-1]
     assert dc_tab.label == "MOS Table"
     assert len(dc_tab.meta) == 0
+
+    # Test all the spectra exist
+    for dispersion in ('R', 'C'):
+        for sourceid in (243, 249):
+            for spec_type in ('spec2d', 'spec1d'):
+                data_label = f"F200W Source {sourceid} {spec_type} {dispersion}"
+                data = mosviz_helper.app.data_collection[data_label]
+                assert data.meta['SOURCEID'] == sourceid
+
+                # Header should be imported from the spec2d files, not spec1d
+                if spec_type == 'spec2d':
+                    assert PRIHDR_KEY in data.meta
+                    assert COMMENTCARD_KEY in data.meta
+                else:
+                    assert PRIHDR_KEY not in data.meta
+                    assert COMMENTCARD_KEY in data.meta
+                    assert 'header' not in data.meta
+                    assert data.meta['FILTER'] == f'GR150{dispersion}'
+
+
+@pytest.mark.remote_data
+def test_missing_srctype(mosviz_helper, tmp_path):
+    '''
+    Tests that data missing the SRCTYPE keyword raises a warning to the user.
+
+    SRCTYPE is required for Mosviz. We do not want to rely on the JWST x1d parser's
+    default behavior of overwriting with "POINT" if it doesn't exist, as all NIRISS data
+    should have this populated; missing SRCTYPE indicates something went wrong.
+
+    This dataset was our original simulated NIRISS dataset that is missing SRCTYPE.
+
+    NOTE: Under some conditions, a warning is raised when TemporaryDirectory attempts to
+    clean itself up. Most likely a race condition between TempDir and pytest
+    '''
+
+    # Download data
+    test_data = 'https://stsci.box.com/shared/static/l2azhcqd3tvzhybdlpx2j2qlutkaro3z.zip'
+    fn = download_file(test_data, cache=True, timeout=30)
+
+    # Extract to a known, temporary folder
+    with ZipFile(fn, 'r') as sample_data_zip:
+        sample_data_zip.extractall(tmp_path)
+
+    with pytest.raises(KeyError, match=r".*The SRCTYPE keyword.*is not populated.*"):
+        mosviz_helper.load_data(directory=(tmp_path / 'NIRISS_for_parser_p0171'),
+                                instrument="niriss")