add support for load_data from URI/URL

CHANGES.rst

@@ -81,6 +81,8 @@ Other Changes and Additions
 New Features
 ------------
 
+- Load remote data from a URI or URL. [#2875]
+
 Cubeviz
 ^^^^^^^
 

jdaviz/app.py

@@ -863,22 +863,8 @@ def load_data(self, file_obj, parser_reference=None, **kwargs):
         """
         self.loading = True
         try:
-            try:
-                # Properly form path and check if a valid file
-                file_obj = pathlib.Path(file_obj)
-                if not file_obj.exists():
-                    msg_text = "Error: File {} does not exist".format(file_obj)
-                    snackbar_message = SnackbarMessage(msg_text, sender=self,
-                                                       color='error')
-                    self.hub.broadcast(snackbar_message)
-                    raise FileNotFoundError("Could not locate file: {}".format(file_obj))
-                else:
-                    # Convert path to properly formatted string (Parsers do not accept path objs)
-                    file_obj = str(file_obj)
-            except TypeError:
-                # If it's not a str/path type, it might be a compatible class.
-                # Pass to parsers to see if they can accept it
-                pass
+            if isinstance(file_obj, pathlib.Path):
+                file_obj = str(file_obj)
 
             # attempt to get a data parser from the config settings
             parser = None

jdaviz/configs/cubeviz/plugins/parsers.py

@@ -12,7 +12,7 @@
 
 from jdaviz.configs.imviz.plugins.parsers import prep_data_layer_as_dq
 from jdaviz.core.registries import data_parser_registry
-from jdaviz.utils import standardize_metadata, PRIHDR_KEY
+from jdaviz.utils import standardize_metadata, PRIHDR_KEY, download_uri_to_path
 
 
 __all__ = ['parse_data']
@@ -23,7 +23,7 @@
 
 
 @data_parser_registry("cubeviz-data-parser")
-def parse_data(app, file_obj, data_type=None, data_label=None, parent=None):
+def parse_data(app, file_obj, data_type=None, data_label=None, parent=None, cache=True):
     """
     Attempts to parse a data file and auto-populate available viewers in
     cubeviz.
@@ -66,6 +66,9 @@ def parse_data(app, file_obj, data_type=None, data_label=None, parent=None):
                        flux_viewer_reference_name=flux_viewer_reference_name)
             return
 
+        # try parsing file_obj as a URI/URL:
+        file_obj = download_uri_to_path(file_obj, cache=cache)
+
         file_name = os.path.basename(file_obj)
 
         with fits.open(file_obj) as hdulist:

jdaviz/configs/cubeviz/plugins/tests/test_parsers.py

@@ -213,7 +213,7 @@ def test_numpy_cube(cubeviz_helper):
 
 
 def test_invalid_data_types(cubeviz_helper):
-    with pytest.raises(FileNotFoundError, match='Could not locate file'):
+    with pytest.raises(FileNotFoundError, match='No such file'):
         cubeviz_helper.load_data('does_not_exist.fits')
 
     with pytest.raises(NotImplementedError, match='Unsupported data format'):

jdaviz/configs/imviz/plugins/parsers.py

@@ -14,7 +14,7 @@
 
 from jdaviz.core.registries import data_parser_registry
 from jdaviz.core.events import SnackbarMessage
-from jdaviz.utils import standardize_metadata, PRIHDR_KEY, _wcs_only_label
+from jdaviz.utils import standardize_metadata, PRIHDR_KEY, _wcs_only_label, download_uri_to_path
 
 try:
     from roman_datamodels import datamodels as rdd
@@ -43,7 +43,7 @@ def prep_data_layer_as_dq(data):
 
 
 @data_parser_registry("imviz-data-parser")
-def parse_data(app, file_obj, ext=None, data_label=None, parent=None):
+def parse_data(app, file_obj, ext=None, data_label=None, parent=None, cache=False):
     """Parse a data file into Imviz.
 
     Parameters
@@ -65,6 +65,9 @@ def parse_data(app, file_obj, ext=None, data_label=None, parent=None):
         if data_label is None:
             data_label = os.path.splitext(os.path.basename(file_obj))[0]
 
+        # try parsing file_obj as a URI/URL:
+        file_obj = download_uri_to_path(file_obj, cache=cache)
+
         # If file_obj is a path to a cached file from
         # astropy.utils.data.download_file, the path has no file extension.
         # Here we check if the file is in the download cache, and if it is,

jdaviz/configs/mosviz/plugins/parsers.py

@@ -16,7 +16,7 @@
 from jdaviz.configs.imviz.plugins.parsers import get_image_data_iterator
 from jdaviz.core.registries import data_parser_registry
 from jdaviz.core.events import SnackbarMessage
-from jdaviz.utils import standardize_metadata, PRIHDR_KEY
+from jdaviz.utils import standardize_metadata, PRIHDR_KEY, download_uri_to_path
 
 __all__ = ['mos_spec1d_parser', 'mos_spec2d_parser', 'mos_image_parser']
 
@@ -259,7 +259,7 @@ def mos_spec1d_parser(app, data_obj, data_labels=None,
 
 @data_parser_registry("mosviz-spec2d-parser")
 def mos_spec2d_parser(app, data_obj, data_labels=None, add_to_table=True,
-                      show_in_viewer=False, ext=1, transpose=False):
+                      show_in_viewer=False, ext=1, transpose=False, cache=True):
     """
     Attempts to parse a 2D spectrum object.
 
@@ -347,6 +347,10 @@ def _parse_as_spectrum1d(hdulist, ext, transpose):
             # If we got a filepath, first try and parse using the Spectrum1D and
             # SpectrumList parsers, and then fall back to parsing it as a generic
             # FITS file.
+
+            # try parsing file_obj as a URI/URL:
+            data = download_uri_to_path(data, cache=cache)
+
             if _check_is_file(data):
                 try:
                     if ext != 1 or transpose:

jdaviz/configs/specviz/plugins/parsers.py

@@ -9,15 +9,15 @@
 
 from jdaviz.core.events import SnackbarMessage
 from jdaviz.core.registries import data_parser_registry
-from jdaviz.utils import standardize_metadata
+from jdaviz.utils import standardize_metadata, download_uri_to_path
 
 
 __all__ = ["specviz_spectrum1d_parser"]
 
 
 @data_parser_registry("specviz-spectrum1d-parser")
 def specviz_spectrum1d_parser(app, data, data_label=None, format=None, show_in_viewer=True,
-                              concat_by_file=False):
+                              concat_by_file=False, cache=False):
     """
     Loads a data file or `~specutils.Spectrum1D` object into Specviz.
 
@@ -58,6 +58,9 @@ def specviz_spectrum1d_parser(app, data, data_label=None, format=None, show_in_v
             # list treated as SpectrumList if not an HDUList
             data = SpectrumList.read(data, format=format)
     else:
+        # try parsing file_obj as a URI/URL:
+        data = download_uri_to_path(data, cache=cache)
+
         path = pathlib.Path(data)
 
         if path.is_file():

jdaviz/tests/test_utils.py

@@ -1,5 +1,8 @@
 import pytest
+import warnings
 
+from astropy.wcs import FITSFixedWarning
+from asdf.exceptions import AsdfWarning
 from jdaviz import utils
 
 
@@ -14,3 +17,39 @@ def test_alpha_index_exceptions():
         utils.alpha_index(4.2)
     with pytest.raises(ValueError, match="index must be positive"):
         utils.alpha_index(-1)
+
+
+@pytest.mark.remote_data
+def test_uri_to_download_imviz(imviz_helper):
+    uri = "mast:HST/product/jezz02ljq_drz.fits"
+    imviz_helper.load_data(uri)
+
+
+@pytest.mark.remote_data
+def test_url_to_download_imviz(imviz_helper):
+    url = "https://www.astropy.org/astropy-data/tutorials/FITS-images/HorseHead.fits"
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', FITSFixedWarning)
+        imviz_helper.load_data(url)
+
+
+@pytest.mark.remote_data
+def test_uri_to_download_cubeviz(cubeviz_helper):
+    uri = "mast:JWST/product/jw01373-o031_t007_miri_ch1-shortmediumlong_s3d.fits"
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', FITSFixedWarning)
+        cubeviz_helper.load_data(uri)
+
+
+@pytest.mark.remote_data
+def test_uri_to_download_specviz(specviz_helper):
+    uri = "mast:JWST/product/jw02732-o004_t004_miri_ch1-shortmediumlong_x1d.fits"
+    specviz_helper.load_data(uri)
+
+
+@pytest.mark.remote_data
+def test_uri_to_download_specviz2d(specviz2d_helper):
+    uri = "mast:JWST/product/jw01324-o006_s00005_nirspec_f100lp-g140h_s2d.fits"
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', AsdfWarning)
+        specviz2d_helper.load_data(uri)

jdaviz/utils.py

@@ -2,19 +2,24 @@
 import time
 import threading
 from collections import deque
+from urllib.parse import urlparse
 
 import numpy as np
 from astropy.io import fits
 from astropy.utils import minversion
+from astropy.utils.data import download_file
 from astropy.wcs.wcsapi import BaseHighLevelWCS
+from astroquery.mast import Observations
+
 from glue.config import settings
 from glue.core import BaseData
 from glue.core.exceptions import IncompatibleAttribute
 from glue.core.subset import SubsetState, RangeSubsetState, RoiSubsetState
 from ipyvue import watch
 
 __all__ = ['SnackbarQueue', 'enable_hot_reloading', 'bqplot_clear_figure',
-           'standardize_metadata', 'ColorCycler', 'alpha_index', 'get_subset_type']
+           'standardize_metadata', 'ColorCycler', 'alpha_index', 'get_subset_type',
+           'download_uri_to_path']
 
 NUMPY_LT_2_0 = not minversion("numpy", "2.0.dev")
 
@@ -385,3 +390,52 @@ def total_masked_first_data(self):
     def __setgluestate__(cls, rec, context):
         masks = {key: context.object(value) for key, value in rec['masks'].items()}
         return cls(masks=masks)
+
+
+def download_uri_to_path(possible_uri, cache=False, local_path=None):
+    """
+    Retrieve data from a URI (or a URL). Return the input if it
+    cannot be parsed as a URI.
+
+    Parameters
+    ----------
+    possible_uri : str or other
+
+    cache: bool, optional
+        Cache file after download. Default is False.
+    local_path : str, optional
+        Save the downloaded file to this path. Default is to
+        save the file with its remote filename in the current
+        working directory.
+
+    Returns
+    -------
+    possible_uri : str or other
+        If ``possible_uri`` cannot be retrieved as a URI, returns the input argument
+        unchanged. If ``possible_uri`` can be retrieved as a URI, returns the
+        local path to the downloaded file.
+    """
+    if not isinstance(possible_uri, str):
+        # only try to parse strings:
+        return possible_uri
+
+    if os.path.exists(possible_uri):
+        # don't try to parse file paths:
+        return possible_uri
+
+    parsed_uri = urlparse(possible_uri)
+
+    if parsed_uri.scheme.lower() == 'mast':
+        Observations.download_file(possible_uri, cache=cache, local_path=local_path)
+
+        if local_path is None:
+            # if not specified, this is the default location:
+            local_path = os.path.join(os.getcwd(), parsed_uri.path.split('/')[-1])
+
+        return local_path
+
+    elif parsed_uri.scheme.lower() in ['http', 'https', 'ftp']:
+        return download_file(possible_uri, cache=cache)
+
+    # assume this isn't a URI after all:
+    return possible_uri