Merge branch 'develop' into telpca

pypeit · Aug 28, 2023 · 009919c · 009919c
2 parents aefd139 + 321cc5f
commit 009919c
Show file tree

Hide file tree

Showing 92 changed files with 921 additions and 608 deletions.
diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
@@ -16,7 +16,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python: ['3.9', '3.10', '3.11']
-        toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev, conda]
+        toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev]
     steps:
     - name: Check out repository
       uses: actions/checkout@v3
@@ -31,4 +31,4 @@ jobs:
         python -m pip install --upgrade pip tox
     - name: Test with tox
       run: |
-        tox -e ${{ matrix.toxenv }}
+        tox -e ${{ matrix.python }}-${{ matrix.toxenv }}
diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml
@@ -18,7 +18,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python: ['3.9', '3.10', '3.11']
-        toxenv: [test, test-alldeps-cov, test-linetoolsdev, test-gingadev, test-astropydev, conda]
+        toxenv: [test, test-alldeps-cov, test-linetoolsdev, test-gingadev, test-astropydev]
     steps:
     - name: Check out repository
       uses: actions/checkout@v3
@@ -31,7 +31,7 @@ jobs:
         python -m pip install --upgrade pip tox
     - name: Test with tox
       run: |
-        tox -e ${{ matrix.toxenv }}
+        tox -e ${{ matrix.python }}-${{ matrix.toxenv }}
     - name: Upload coverage to codecov
       if: "contains(matrix.toxenv, '-cov')"
       uses: codecov/codecov-action@v3
@@ -62,7 +62,22 @@ jobs:
         python -m pip install --upgrade pip tox
     - name: Test with tox
       run: |
-        tox -e ${{ matrix.toxenv }}
+        tox -e ${{ matrix.python }}-${{ matrix.toxenv }}
+
+  conda:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Conda environment check
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+    - name: Install base dependencies
+      run: |
+        python -m pip install --upgrade pip tox
+    - name: Run pypeit tests from environment built with conda
+      run: |
+        tox -e conda
 
   codestyle:
     runs-on: ubuntu-latest

diff --git a/LICENSE.rst b/LICENSE.rst
@@ -1,4 +1,4 @@
-Copyright (c) 2018-2019, PypeIt Developers
+Copyright (c) 2018-2023, PypeIt Developers
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

diff --git a/README.rst b/README.rst
@@ -97,7 +97,7 @@ follow our `Code of Conduct
 
 Along with our extensive `online documentation
 <https://pypeit.readthedocs.io/en/release/>`__, we encourage the PypeIt user
-base to communicate via our `PypeIt Users Slack <pypeit-users.slack.com>`__.
+base to communicate via our `PypeIt Users Slack <https://pypeit-users.slack.com>`__.
 All are welcome to join using `this invitation link <https://join.slack.com/t/pypeit-users/shared_invite/zt-1kc4rxhsj-vKU1JnUA~8PZE~tPlu~aTg>`__.
 
 If you find a bug (particularly one that is experienced by others in the Users
@@ -171,4 +171,5 @@ development of PypeIt.
 * Timothy Pickering
 * Timothy Ellsworth-Bowers
 * Gregory Simonian
-* Heather Martin
+* Heather Martin
+
diff --git a/doc/Makefile b/doc/Makefile
@@ -84,7 +84,7 @@ htmlonly:
 
 picky:
 	make apirst
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) -n $(BUILDDIR)/html
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) -n -vvv -T $(BUILDDIR)/html -W
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 

diff --git a/doc/calibrations/calibrations.rst b/doc/calibrations/calibrations.rst
@@ -1,4 +1,6 @@
 
+.. include:: ../include/links.rst
+
 .. _calibrations:
 
 ============

diff --git a/doc/calibrations/wave_calib.rst b/doc/calibrations/wave_calib.rst
@@ -270,13 +270,19 @@ grating equation.
 
 In general, the approach is:
 
-    1. Identify the arc lines in each order
-    2. Fit the arc lines in each order to a polynomial, individually
-    3. Fit a 2D solution to the lines using the order number as a basis
-    4. Reject orders where the RMS of the fit (measured in binned pixels) exceeds ``rms_threshold``
-    5. Attempt to recover the missing orders using the 2D fit and 
-      a higher RMS threshold
-    6. Refit the 2D solution
+    #. Identify the arc lines in each order
+
+    #. Fit the arc lines in each order to a polynomial, individually
+
+    #. Fit a 2D solution to the lines using the order number as a basis
+
+    #. Reject orders where the RMS of the fit (measured in binned pixels)
+       exceeds ``rms_threshold``
+
+    #. Attempt to recover the missing orders using the 2D fit and a higher RMS
+       threshold
+
+    #. Refit the 2D solution
 
 One should always inspect the outputs, especially the 2D solution
 (global and orders).  One may then need to modify the ``rms_threshold``

diff --git a/doc/coadd2d.rst b/doc/coadd2d.rst
@@ -28,13 +28,13 @@ during reductions (see :ref:`2d_combine`).
         ``1800``, ``1800``, ``1800``, and ``1200`` seconds, the exposure
         time of the coadded frame will be:
 
-        .. code-block:: ini
+        .. code-block:: python
 
             coadd_exptime = np.percentile([1800,1800,1800,1200],50, method='higher')
 
         and the flux of the individual frames will be rescaled by:
 
-        .. code-block:: ini
+        .. code-block:: python
 
             rescale_factor = coadd_exptime / exptime
 

diff --git a/doc/collate1d.rst b/doc/collate1d.rst
@@ -144,6 +144,11 @@ followed by a list of spec1d files. An example configuration file is shown below
     # current directory.
     #outdir = /work/output
     
+    # Whether to check that spec1d files and archival sensfunc files have an
+    # up to date datamodel version. If false (the default) version numbers are 
+    # not checked.
+    #chk_version = True
+
     # A list of the spec1d files. Wildcards are allowed.
     # This follows the input file data block format.
     spec1d read

diff --git a/doc/conf.py b/doc/conf.py
@@ -349,5 +349,5 @@
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
 
diff --git a/doc/cookbook.rst b/doc/cookbook.rst
@@ -33,7 +33,7 @@ Note that:
 
   - Invariably something will be out of date in our doc pages.  When you see an
     egregious example, please holler on `GitHub
-    <https://github.com/pypeit/PypeIt>`__ or `Slack <pypeit-users.slack.com>`__.
+    <https://github.com/pypeit/PypeIt>`__ or `Slack <https://pypeit-users.slack.com>`__.
 
 Finally, note that before you keep going, you should have already done the following:
 

diff --git a/doc/help/pypeit_collate_1d.rst b/doc/help/pypeit_collate_1d.rst
@@ -8,7 +8,7 @@
                              [--flux] [--exclude_slit_bm EXCLUDE_SLIT_BM]
                              [--exclude_serendip] [--wv_rms_thresh WV_RMS_THRESH]
                              [--refframe {observed,heliocentric,barycentric}]
-                             [-v VERBOSITY]
+                             [--chk_version] [-v VERBOSITY]
                              [input_file]
     
     Flux/Coadd multiple 1d spectra from multiple nights and prepare a directory for
@@ -36,6 +36,8 @@
                                                     value are skipped, else all wavelength rms values are accepted.
                               refframe              Perform reference frame correction prior to coadding.
                                                     Options are ['observed', 'heliocentric', 'barycentric']. Defaults to None.
+                              chk_version           If true, spec1ds and archival sensfuncs must match the currently
+                                                    supported versions. If false (the default) version numbers are not checked.
                              
                             spec1d read
                             <path to spec1d files, wildcards allowed>
@@ -86,6 +88,8 @@
       --refframe {observed,heliocentric,barycentric}
                             Perform reference frame correction prior to coadding.
                             Options are: observed, heliocentric, barycentric
+      --chk_version         Whether to check the data model versions of spec1d files
+                            and sensfunc files.
       -v VERBOSITY, --verbosity VERBOSITY
                             Verbosity level between 0 [none] and 2 [all]. Default:
                             1. Level 2 writes a log with filename

diff --git a/doc/help/pypeit_sensfunc.rst b/doc/help/pypeit_sensfunc.rst
@@ -69,7 +69,7 @@
                             the flatfile there via e.g.:
                              
                                 [sensfunc]
-                                     flatfile = Calibrations/Flat_A_0_DET01.fits'
+                                     flatfile = Calibrations/Flat_A_0_DET01.fits
                              
                             Where Flat_A_0_DET01.fits is the flat file in your
                             Calibrations directory

diff --git a/doc/help/pypeit_setup_coadd2d.rst b/doc/help/pypeit_setup_coadd2d.rst
@@ -42,7 +42,7 @@
                             --det 1,5 3,7 (default: None)
       --only_slits ONLY_SLITS [ONLY_SLITS ...]
                             A space-separated set of slits to coadd. Example syntax
-                            -- --only_slits DET01:175,DET02:205 or MSC02:2234. If
+                            for argument is DET01:175,DET02:205 or MSC02:2234. If
                             not provided, all slits are coadded. If both --det and
                             --only_slits are provided, --det will be ignored. This
                             and --exclude_slits are mutually exclusive. If both are

diff --git a/doc/help/pypeit_view_fits.rst b/doc/help/pypeit_view_fits.rst
@@ -35,9 +35,8 @@
       -h, --help           show this help message and exit
       --list               List the extensions only? (default: False)
       --proc               Process the image (i.e. orient, overscan subtract,
-                           multiply by gain) using pypeit.images.buildimage. Note
-                           det=mosaic will not work with this option (default:
-                           False)
+                           multiply by gain) using pypeit.images.buildimage.
+                           (default: False)
       --bkg_file BKG_FILE  FITS file to be subtracted from the image in file.--proc
                            must be set in order for this option to work. (default:
                            None)

diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst
@@ -4,7 +4,7 @@
     usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c]
                       pypeit_file
     
-    ##  [1;37;42mPypeIt : The Python Spectroscopic Data Reduction Pipeline v1.13.1.dev537+g0261baa1b.d20230728[0m
+    ##  [1;37;42mPypeIt : The Python Spectroscopic Data Reduction Pipeline v1.13.1.dev562+g146ed41eb.d20230809[0m
     ##  
     ##  Available spectrographs include:
     ##   bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v,

diff --git a/doc/include/class_datamodel_datacube.rst b/doc/include/class_datamodel_datacube.rst
@@ -1,15 +1,15 @@
 
 **Version**: 1.1.0
 
-==============  ================  =================  ================================================================================
-Attribute       Type              Array Type         Description                                                                     
-==============  ================  =================  ================================================================================
-``PYP_SPEC``    str                                  PypeIt: Spectrograph name                                                       
-``blaze_spec``  `numpy.ndarray`_  `numpy.floating`_  The spectral blaze function                                                     
-``blaze_wave``  `numpy.ndarray`_  `numpy.floating`_  Wavelength array of the spectral blaze function                                 
-``bpm``         `numpy.ndarray`_  `numpy.uint8`_     Bad pixel mask of the datacube (0=good, 1=bad)                                  
-``flux``        `numpy.ndarray`_  `numpy.floating`_  Flux datacube in units of counts/s/Ang/arcsec^2or 10^-17 erg/s/cm^2/Ang/arcsec^2
-``fluxed``      bool                                 Boolean indicating if the datacube is fluxed.                                   
-``sensfunc``    `numpy.ndarray`_  `numpy.floating`_  Sensitivity function 10^-17 erg/(counts/cm^2)                                   
-``sig``         `numpy.ndarray`_  `numpy.floating`_  Error datacube (matches units of flux)                                          
-==============  ================  =================  ================================================================================
+==============  ================  =================  =================================================================================
+Attribute       Type              Array Type         Description                                                                      
+==============  ================  =================  =================================================================================
+``PYP_SPEC``    str                                  PypeIt: Spectrograph name                                                        
+``blaze_spec``  `numpy.ndarray`_  `numpy.floating`_  The spectral blaze function                                                      
+``blaze_wave``  `numpy.ndarray`_  `numpy.floating`_  Wavelength array of the spectral blaze function                                  
+``bpm``         `numpy.ndarray`_  `numpy.uint8`_     Bad pixel mask of the datacube (0=good, 1=bad)                                   
+``flux``        `numpy.ndarray`_  `numpy.floating`_  Flux datacube in units of counts/s/Ang/arcsec^2 or 10^-17 erg/s/cm^2/Ang/arcsec^2
+``fluxed``      bool                                 Boolean indicating if the datacube is fluxed.                                    
+``sensfunc``    `numpy.ndarray`_  `numpy.floating`_  Sensitivity function 10^-17 erg/(counts/cm^2)                                    
+``sig``         `numpy.ndarray`_  `numpy.floating`_  Error datacube (matches units of flux)                                           
+==============  ================  =================  =================================================================================
diff --git a/doc/include/dependencies_table.rst b/doc/include/dependencies_table.rst
@@ -1,5 +1,5 @@
 =======================  =================================================================================================================================================================================================================================================================================================
 Python Version           ``>=3.9,<3.12``                                                                                                                                                                                                                                                                                  
 Required for users       ``IPython>=7.10.0``, ``PyYAML>=5.1``, ``astropy>=4.3``, ``bottleneck``, ``configobj>=5.0.6``, ``extension-helpers>=0.1``, ``ginga>=3.2``, ``linetools``, ``matplotlib>=3.7``, ``numpy>=1.22``, ``packaging>=0.19``, ``pygithub``, ``pyqt6``, ``qtpy>=1.9``, ``scikit-learn>=1.0``, ``scipy>=1.7``
-Required for developers  ``coverage``, ``docutils<0.18``, ``pytest-astropy``, ``pytest-cov``, ``pytest>=6.0.0``, ``sphinx-automodapi``, ``sphinx<6,>=1.6``, ``sphinx_rtd_theme==1.1.1``, ``tox``                                                                                                                          
+Required for developers  ``coverage``, ``docutils<0.19``, ``pytest-astropy``, ``pytest-cov``, ``pytest>=6.0.0``, ``scikit-image``, ``specutils``, ``sphinx-automodapi``, ``sphinx<7,>=1.6``, ``sphinx_rtd_theme==1.2.2``, ``tox``                                                                                         
 =======================  =================================================================================================================================================================================================================================================================================================
diff --git a/doc/include/gemini_gnirs_echelle_A_corrected.pypeit.rst b/doc/include/gemini_gnirs_echelle_A_corrected.pypeit.rst
@@ -6,10 +6,10 @@
     # User-defined execution parameters
     [rdx]
     spectrograph = gemini_gnirs_echelle
-    [calibrations]
-     [[wavelengths]]
-        rms_threshold = 0.8, 0.4, 0.8, 0.8, 0.5, 0.9
-        sigdetect = 5.,5.,6.,6.,5.,7.
+    #[calibrations]
+    # [[wavelengths]]
+    #    rms_threshold = 0.8, 0.4, 0.8, 0.8, 0.5, 0.9
+    #    sigdetect = 5.,5.,6.,6.,5.,7.
     
     # Setup
     setup read

diff --git a/doc/include/links.rst b/doc/include/links.rst
@@ -35,7 +35,6 @@
 .. scipy
 .. _scipy.optimize.curve_fit: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html
 .. _scipy.optimize.leastsq: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.leastsq.html
-.. _scipy.interpolate.interp1d: https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.interp1d.html
 .. _scipy.optimize.least_squares: http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.least_squares.html
 .. _scipy.optimize.OptimizeResult: http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.OptimizeResult.html
 .. _scipy.optimize.differential_evolution: https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.differential_evolution.html
@@ -155,3 +154,5 @@
 .. _L.A.Cosmic: http://www.astro.yale.edu/dokkum/lacosmic/
 .. _van Dokkum (2001, PASP, 113, 1420): https://ui.adsabs.harvard.edu/abs/2001PASP..113.1420V/abstract
 .. _LowRedux: http://www.ucolick.org/~xavier/LowRedux/
+
+