From 770e738104195a5a34caf3bfdfc81d57eba1452b Mon Sep 17 00:00:00 2001 From: "Brett M. Morris" Date: Wed, 15 May 2024 11:55:14 -0400 Subject: [PATCH] add support for load_data from URI/URL --- CHANGES.rst | 2 + jdaviz/app.py | 18 +----- jdaviz/configs/cubeviz/plugins/parsers.py | 7 ++- .../cubeviz/plugins/tests/test_parsers.py | 2 +- jdaviz/configs/imviz/plugins/parsers.py | 7 ++- jdaviz/configs/mosviz/plugins/parsers.py | 8 ++- jdaviz/configs/specviz/plugins/parsers.py | 7 ++- jdaviz/tests/test_utils.py | 39 +++++++++++++ jdaviz/utils.py | 56 ++++++++++++++++++- 9 files changed, 120 insertions(+), 26 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 29095a5a6a..9eb631d226 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -81,6 +81,8 @@ Other Changes and Additions New Features ------------ +- Load remote data from a URI or URL. [#2875] + Cubeviz ^^^^^^^ diff --git a/jdaviz/app.py b/jdaviz/app.py index e65de0270e..03b5efd789 100644 --- a/jdaviz/app.py +++ b/jdaviz/app.py @@ -863,22 +863,8 @@ def load_data(self, file_obj, parser_reference=None, **kwargs): """ self.loading = True try: - try: - # Properly form path and check if a valid file - file_obj = pathlib.Path(file_obj) - if not file_obj.exists(): - msg_text = "Error: File {} does not exist".format(file_obj) - snackbar_message = SnackbarMessage(msg_text, sender=self, - color='error') - self.hub.broadcast(snackbar_message) - raise FileNotFoundError("Could not locate file: {}".format(file_obj)) - else: - # Convert path to properly formatted string (Parsers do not accept path objs) - file_obj = str(file_obj) - except TypeError: - # If it's not a str/path type, it might be a compatible class. - # Pass to parsers to see if they can accept it - pass + if isinstance(file_obj, pathlib.Path): + file_obj = str(file_obj) # attempt to get a data parser from the config settings parser = None diff --git a/jdaviz/configs/cubeviz/plugins/parsers.py b/jdaviz/configs/cubeviz/plugins/parsers.py index 7c7b14f2b4..90be7ca0c0 100644 --- a/jdaviz/configs/cubeviz/plugins/parsers.py +++ b/jdaviz/configs/cubeviz/plugins/parsers.py @@ -12,7 +12,7 @@ from jdaviz.configs.imviz.plugins.parsers import prep_data_layer_as_dq from jdaviz.core.registries import data_parser_registry -from jdaviz.utils import standardize_metadata, PRIHDR_KEY +from jdaviz.utils import standardize_metadata, PRIHDR_KEY, download_uri_to_path __all__ = ['parse_data'] @@ -23,7 +23,7 @@ @data_parser_registry("cubeviz-data-parser") -def parse_data(app, file_obj, data_type=None, data_label=None, parent=None): +def parse_data(app, file_obj, data_type=None, data_label=None, parent=None, cache=True): """ Attempts to parse a data file and auto-populate available viewers in cubeviz. @@ -66,6 +66,9 @@ def parse_data(app, file_obj, data_type=None, data_label=None, parent=None): flux_viewer_reference_name=flux_viewer_reference_name) return + # try parsing file_obj as a URI/URL: + file_obj = download_uri_to_path(file_obj, cache=cache) + file_name = os.path.basename(file_obj) with fits.open(file_obj) as hdulist: diff --git a/jdaviz/configs/cubeviz/plugins/tests/test_parsers.py b/jdaviz/configs/cubeviz/plugins/tests/test_parsers.py index 9c5206a811..85d17f31e8 100644 --- a/jdaviz/configs/cubeviz/plugins/tests/test_parsers.py +++ b/jdaviz/configs/cubeviz/plugins/tests/test_parsers.py @@ -213,7 +213,7 @@ def test_numpy_cube(cubeviz_helper): def test_invalid_data_types(cubeviz_helper): - with pytest.raises(FileNotFoundError, match='Could not locate file'): + with pytest.raises(FileNotFoundError, match='No such file'): cubeviz_helper.load_data('does_not_exist.fits') with pytest.raises(NotImplementedError, match='Unsupported data format'): diff --git a/jdaviz/configs/imviz/plugins/parsers.py b/jdaviz/configs/imviz/plugins/parsers.py index d0a17a321a..5d285e1995 100644 --- a/jdaviz/configs/imviz/plugins/parsers.py +++ b/jdaviz/configs/imviz/plugins/parsers.py @@ -14,7 +14,7 @@ from jdaviz.core.registries import data_parser_registry from jdaviz.core.events import SnackbarMessage -from jdaviz.utils import standardize_metadata, PRIHDR_KEY, _wcs_only_label +from jdaviz.utils import standardize_metadata, PRIHDR_KEY, _wcs_only_label, download_uri_to_path try: from roman_datamodels import datamodels as rdd @@ -43,7 +43,7 @@ def prep_data_layer_as_dq(data): @data_parser_registry("imviz-data-parser") -def parse_data(app, file_obj, ext=None, data_label=None, parent=None): +def parse_data(app, file_obj, ext=None, data_label=None, parent=None, cache=False): """Parse a data file into Imviz. Parameters @@ -65,6 +65,9 @@ def parse_data(app, file_obj, ext=None, data_label=None, parent=None): if data_label is None: data_label = os.path.splitext(os.path.basename(file_obj))[0] + # try parsing file_obj as a URI/URL: + file_obj = download_uri_to_path(file_obj, cache=cache) + # If file_obj is a path to a cached file from # astropy.utils.data.download_file, the path has no file extension. # Here we check if the file is in the download cache, and if it is, diff --git a/jdaviz/configs/mosviz/plugins/parsers.py b/jdaviz/configs/mosviz/plugins/parsers.py index 88bc620216..192a022a5e 100644 --- a/jdaviz/configs/mosviz/plugins/parsers.py +++ b/jdaviz/configs/mosviz/plugins/parsers.py @@ -16,7 +16,7 @@ from jdaviz.configs.imviz.plugins.parsers import get_image_data_iterator from jdaviz.core.registries import data_parser_registry from jdaviz.core.events import SnackbarMessage -from jdaviz.utils import standardize_metadata, PRIHDR_KEY +from jdaviz.utils import standardize_metadata, PRIHDR_KEY, download_uri_to_path __all__ = ['mos_spec1d_parser', 'mos_spec2d_parser', 'mos_image_parser'] @@ -259,7 +259,7 @@ def mos_spec1d_parser(app, data_obj, data_labels=None, @data_parser_registry("mosviz-spec2d-parser") def mos_spec2d_parser(app, data_obj, data_labels=None, add_to_table=True, - show_in_viewer=False, ext=1, transpose=False): + show_in_viewer=False, ext=1, transpose=False, cache=True): """ Attempts to parse a 2D spectrum object. @@ -347,6 +347,10 @@ def _parse_as_spectrum1d(hdulist, ext, transpose): # If we got a filepath, first try and parse using the Spectrum1D and # SpectrumList parsers, and then fall back to parsing it as a generic # FITS file. + + # try parsing file_obj as a URI/URL: + data = download_uri_to_path(data, cache=cache) + if _check_is_file(data): try: if ext != 1 or transpose: diff --git a/jdaviz/configs/specviz/plugins/parsers.py b/jdaviz/configs/specviz/plugins/parsers.py index 23a794dd52..a231e593a3 100644 --- a/jdaviz/configs/specviz/plugins/parsers.py +++ b/jdaviz/configs/specviz/plugins/parsers.py @@ -9,7 +9,7 @@ from jdaviz.core.events import SnackbarMessage from jdaviz.core.registries import data_parser_registry -from jdaviz.utils import standardize_metadata +from jdaviz.utils import standardize_metadata, download_uri_to_path __all__ = ["specviz_spectrum1d_parser"] @@ -17,7 +17,7 @@ @data_parser_registry("specviz-spectrum1d-parser") def specviz_spectrum1d_parser(app, data, data_label=None, format=None, show_in_viewer=True, - concat_by_file=False): + concat_by_file=False, cache=False): """ Loads a data file or `~specutils.Spectrum1D` object into Specviz. @@ -58,6 +58,9 @@ def specviz_spectrum1d_parser(app, data, data_label=None, format=None, show_in_v # list treated as SpectrumList if not an HDUList data = SpectrumList.read(data, format=format) else: + # try parsing file_obj as a URI/URL: + data = download_uri_to_path(data, cache=cache) + path = pathlib.Path(data) if path.is_file(): diff --git a/jdaviz/tests/test_utils.py b/jdaviz/tests/test_utils.py index c243c1a714..b4596fe2c8 100644 --- a/jdaviz/tests/test_utils.py +++ b/jdaviz/tests/test_utils.py @@ -1,5 +1,8 @@ import pytest +import warnings +from astropy.wcs import FITSFixedWarning +from asdf.exceptions import AsdfWarning from jdaviz import utils @@ -14,3 +17,39 @@ def test_alpha_index_exceptions(): utils.alpha_index(4.2) with pytest.raises(ValueError, match="index must be positive"): utils.alpha_index(-1) + + +@pytest.mark.remote_data +def test_uri_to_download_imviz(imviz_helper): + uri = "mast:HST/product/jezz02ljq_drz.fits" + imviz_helper.load_data(uri) + + +@pytest.mark.remote_data +def test_url_to_download_imviz(imviz_helper): + url = "https://www.astropy.org/astropy-data/tutorials/FITS-images/HorseHead.fits" + with warnings.catch_warnings(): + warnings.simplefilter('ignore', FITSFixedWarning) + imviz_helper.load_data(url) + + +@pytest.mark.remote_data +def test_uri_to_download_cubeviz(cubeviz_helper): + uri = "mast:JWST/product/jw01373-o031_t007_miri_ch1-shortmediumlong_s3d.fits" + with warnings.catch_warnings(): + warnings.simplefilter('ignore', FITSFixedWarning) + cubeviz_helper.load_data(uri) + + +@pytest.mark.remote_data +def test_uri_to_download_specviz(specviz_helper): + uri = "mast:JWST/product/jw02732-o004_t004_miri_ch1-shortmediumlong_x1d.fits" + specviz_helper.load_data(uri) + + +@pytest.mark.remote_data +def test_uri_to_download_specviz2d(specviz2d_helper): + uri = "mast:JWST/product/jw01324-o006_s00005_nirspec_f100lp-g140h_s2d.fits" + with warnings.catch_warnings(): + warnings.simplefilter('ignore', AsdfWarning) + specviz2d_helper.load_data(uri) diff --git a/jdaviz/utils.py b/jdaviz/utils.py index 448f9ba4bf..3e1a157b23 100644 --- a/jdaviz/utils.py +++ b/jdaviz/utils.py @@ -2,11 +2,15 @@ import time import threading from collections import deque +from urllib.parse import urlparse import numpy as np from astropy.io import fits from astropy.utils import minversion +from astropy.utils.data import download_file from astropy.wcs.wcsapi import BaseHighLevelWCS +from astroquery.mast import Observations + from glue.config import settings from glue.core import BaseData from glue.core.exceptions import IncompatibleAttribute @@ -14,7 +18,8 @@ from ipyvue import watch __all__ = ['SnackbarQueue', 'enable_hot_reloading', 'bqplot_clear_figure', - 'standardize_metadata', 'ColorCycler', 'alpha_index', 'get_subset_type'] + 'standardize_metadata', 'ColorCycler', 'alpha_index', 'get_subset_type', + 'download_uri_to_path'] NUMPY_LT_2_0 = not minversion("numpy", "2.0.dev") @@ -385,3 +390,52 @@ def total_masked_first_data(self): def __setgluestate__(cls, rec, context): masks = {key: context.object(value) for key, value in rec['masks'].items()} return cls(masks=masks) + + +def download_uri_to_path(possible_uri, cache=False, local_path=None): + """ + Retrieve data from a URI (or a URL). Return the input if it + cannot be parsed as a URI. + + Parameters + ---------- + possible_uri : str or other + + cache: bool, optional + Cache file after download. Default is False. + local_path : str, optional + Save the downloaded file to this path. Default is to + save the file with its remote filename in the current + working directory. + + Returns + ------- + possible_uri : str or other + If ``possible_uri`` cannot be retrieved as a URI, returns the input argument + unchanged. If ``possible_uri`` can be retrieved as a URI, returns the + local path to the downloaded file. + """ + if not isinstance(possible_uri, str): + # only try to parse strings: + return possible_uri + + if os.path.exists(possible_uri): + # don't try to parse file paths: + return possible_uri + + parsed_uri = urlparse(possible_uri) + + if parsed_uri.scheme.lower() == 'mast': + Observations.download_file(possible_uri, cache=cache, local_path=local_path) + + if local_path is None: + # if not specified, this is the default location: + local_path = os.path.join(os.getcwd(), parsed_uri.path.split('/')[-1]) + + return local_path + + elif parsed_uri.scheme.lower() in ['http', 'https', 'ftp']: + return download_file(possible_uri, cache=cache) + + # assume this isn't a URI after all: + return possible_uri