Merge branch 'master' into 416-s3-awareness

docs/source/conf.py

@@ -25,7 +25,7 @@
 author = 'The PySPEDAS Community'
 
 # The full version, including alpha/beta/rc tags
-release = '1.7.0'
+release = '1.7.1'
 
 
 # -- General configuration ---------------------------------------------------

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pyspedas"
-version = "1.7.0"
+version = "1.7.1"
 description = "Python Space Physics Environment Data Analysis Software (pySPEDAS)"
 readme = {file = "README.md", content-type = "text/markdown"}
 requires-python = ">=3.9"

pyspedas/projects/themis/config.py

@@ -1,7 +1,7 @@
 import os
 
 CONFIG = {'local_data_dir': 'themis_data/',
-          'remote_data_dir': 'http://themis.ssl.berkeley.edu/data/themis/'}
+          'remote_data_dir': 'https://themis.ssl.berkeley.edu/data/themis/'}
 
 # override local data directory with environment variables
 if os.environ.get('SPEDAS_DATA_DIR'):

pyspedas/utilities/download.py

@@ -333,7 +333,8 @@ def download_file(
         return None
 
     if needs_to_download_file:
-        ftmp = NamedTemporaryFile(delete=False)
+        fsuffix = filename.split('.')[-1] # could be fsspec uri
+        ftmp = NamedTemporaryFile(delete=False, suffix=fsuffix)
 
         with open(ftmp.name, "wb") as f:
             if text_only:
@@ -344,9 +345,14 @@ def download_file(
         if is_fsspec_uri(filename):
             protocol, path = filename.split("://")
             fs = fsspec.filesystem(protocol, anon=False)
-
+            
             # copy method is within filesystems under fsspec
-            fs.put(ftmp.name, filename)
+            if check_downloaded_file(ftmp.name):
+                fs.put(ftmp.name, filename)
+                logging.info("Download complete: " + filename)
+            else:
+                logging.error("Download of '" + filename + "' failed. The temp file will be removed.")
+                logging.error("If the same file has been already downloaded previously, it might be possible to use that instead.")
         else:
             # make sure the directory exists
             if (
@@ -356,13 +362,18 @@ def download_file(
                 os.makedirs(os.path.dirname(filename))
 
             # if the download was successful, copy to data directory
-            copy(ftmp.name, filename)
+            if check_downloaded_file(ftmp.name):
+                copy(ftmp.name, filename)
+                logging.info("Download complete: " + filename)
+            else:
+                logging.error("Download of '" + filename + "' failed. The temp file will be removed.")
+                logging.error("If the same file has been already downloaded previously, it might be possible to use that instead.")
 
+        # cleanup
         fsrc.close()
         ftmp.close()
         os.unlink(ftmp.name)  # delete the temporary file
-
-        logging.info("Download complete: " + filename)
+
 
     # At this point, we check if the file can be opened.
     # If it cannot be opened, we delete the file and try again.