diff --git a/deprecated/arc_old.py b/deprecated/arc_old.py index edecb94dd2..79b58ded49 100644 --- a/deprecated/arc_old.py +++ b/deprecated/arc_old.py @@ -855,3 +855,53 @@ def saturation_mask(a, satlevel): return mask.astype(int) +def mask_around_peaks(spec, inbpm): + """ + Find peaks in the input spectrum and mask pixels around them. + + All pixels to the left and right of a peak is masked until + a pixel has a lower value than the adjacent pixel. At this + point, we assume that spec has reached the noise level. + + Parameters + ---------- + spec: `numpy.ndarray`_ + Spectrum (1D array) in counts + inbpm: `numpy.ndarray`_ + Input bad pixel mask + + Returns + ------- + outbpm: `numpy.ndarray`_ + Bad pixel mask with pixels around peaks masked + """ + # Find the peak locations + pks = detect_peaks(spec) + + # Initialise some useful variables and the output bpm + xarray = np.arange(spec.size) + specdiff = np.append(np.diff(spec), 0.0) + outbpm = inbpm.copy() + + # Loop over the peaks and mask pixels around them + for i in range(len(pks)): + # Find all pixels to the left of the peak that are above the noise level + wl = np.where((xarray <= pks[i]) & (specdiff > 0.0))[0] + ww = (pks[i]-wl)[::-1] + # Find the first pixel to the left of the peak that is below the noise level + nmask = np.where(np.diff(ww) > 1)[0] + if nmask.size != 0 and nmask[0] > 5: + # Mask all pixels to the left of the peak + mini = max(0,wl.size-nmask[0]-1) + outbpm[wl[mini]:pks[i]] = True + # Find all pixels to the right of the peak that are above the noise level + ww = np.where((xarray >= pks[i]) & (specdiff < 0.0))[0] + # Find the first pixel to the right of the peak that is below the noise level + nmask = np.where(np.diff(ww) > 1)[0] + if nmask.size != 0 and nmask[0] > 5: + # Mask all pixels to the right of the peak + maxi = min(nmask[0], ww.size) + outbpm[pks[i]:ww[maxi]+2] = True + # Return the output bpm + return outbpm + diff --git a/doc/api/pypeit.spectrographs.aat_uhrf.rst b/doc/api/pypeit.spectrographs.aat_uhrf.rst new file mode 100644 index 0000000000..2f4e657404 --- /dev/null +++ b/doc/api/pypeit.spectrographs.aat_uhrf.rst @@ -0,0 +1,8 @@ +pypeit.spectrographs.aat\_uhrf module +===================================== + +.. automodule:: pypeit.spectrographs.aat_uhrf + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.spectrographs.rst b/doc/api/pypeit.spectrographs.rst index c53968bcbb..239d591126 100644 --- a/doc/api/pypeit.spectrographs.rst +++ b/doc/api/pypeit.spectrographs.rst @@ -7,6 +7,7 @@ Submodules .. toctree:: :maxdepth: 4 + pypeit.spectrographs.aat_uhrf pypeit.spectrographs.bok_bc pypeit.spectrographs.gemini_flamingos pypeit.spectrographs.gemini_gmos diff --git a/doc/cookbook.rst b/doc/cookbook.rst index 915aed4865..19b397281e 100644 --- a/doc/cookbook.rst +++ b/doc/cookbook.rst @@ -70,7 +70,8 @@ what we recommend: - We will refer to that folder as ``RAWDIR`` The raw images can be gzip-compressed, although this means opening files will be -slower. +slower. See :ref:`setup-file-searching` for specific comments about the files +in your raw directory. A word on calibration data -------------------------- diff --git a/doc/help/pypeit_cache_github_data.rst b/doc/help/pypeit_cache_github_data.rst index 293f6cad17..3ab386e7fd 100644 --- a/doc/help/pypeit_cache_github_data.rst +++ b/doc/help/pypeit_cache_github_data.rst @@ -9,7 +9,7 @@ Script to download/cache PypeIt github data positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, diff --git a/doc/help/pypeit_chk_for_calibs.rst b/doc/help/pypeit_chk_for_calibs.rst index 06cc2d6357..653b5d1e81 100644 --- a/doc/help/pypeit_chk_for_calibs.rst +++ b/doc/help/pypeit_chk_for_calibs.rst @@ -13,7 +13,7 @@ options: -h, --help show this help message and exit -s SPECTROGRAPH, --spectrograph SPECTROGRAPH - A valid spectrograph identifier: bok_bc, + A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -35,8 +35,11 @@ vlt_xshooter_nir, vlt_xshooter_uvb, vlt_xshooter_vis, wht_isis_blue, wht_isis_red (default: None) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) --save_setups If not toggled, remove setup_files/ folder and its files. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_obslog.rst b/doc/help/pypeit_obslog.rst index 3b4c807e8c..5b399fcee0 100644 --- a/doc/help/pypeit_obslog.rst +++ b/doc/help/pypeit_obslog.rst @@ -10,7 +10,7 @@ using PypeItMetaData. positional arguments: - spec A valid spectrograph identifier: bok_bc, + spec A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -75,8 +75,11 @@ -s SORT, --sort SORT Metadata keyword (pypeit-specific) to use to sort the output table. (default: mjd) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) -d OUTPUT_PATH, --output_path OUTPUT_PATH Path to top-level output directory. (default: current working directory) diff --git a/doc/help/pypeit_ql.rst b/doc/help/pypeit_ql.rst index b8991c356d..9302bc39c0 100644 --- a/doc/help/pypeit_ql.rst +++ b/doc/help/pypeit_ql.rst @@ -17,7 +17,7 @@ Script to produce quick-look PypeIt reductions positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, diff --git a/doc/help/pypeit_setup.rst b/doc/help/pypeit_setup.rst index 32f4da9915..01e0c078dd 100644 --- a/doc/help/pypeit_setup.rst +++ b/doc/help/pypeit_setup.rst @@ -11,7 +11,7 @@ options: -h, --help show this help message and exit -s SPECTROGRAPH, --spectrograph SPECTROGRAPH - A valid spectrograph identifier: bok_bc, + A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -39,8 +39,11 @@ --extension option to set the types of files to search for. (default: current working directory) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) -d OUTPUT_PATH, --output_path OUTPUT_PATH Path to top-level output directory. (default: current working directory) diff --git a/doc/help/pypeit_trace_edges.rst b/doc/help/pypeit_trace_edges.rst index dc265db21b..a9b5e21814 100644 --- a/doc/help/pypeit_trace_edges.rst +++ b/doc/help/pypeit_trace_edges.rst @@ -26,16 +26,16 @@ default mosaic. (default: None) -s SPECTROGRAPH, --spectrograph SPECTROGRAPH A valid spectrograph identifier, which is only used if - providing files directly: bok_bc, gemini_flamingos1, - gemini_flamingos2, gemini_gmos_north_e2v, - gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - gemini_gmos_south_ham, gemini_gnirs_echelle, - gemini_gnirs_ifu, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_esi, - keck_hires, keck_kcrm, keck_kcwi, keck_lris_blue, - keck_lris_blue_orig, keck_lris_red, keck_lris_red_mark4, - keck_lris_red_orig, keck_mosfire, keck_nires, - keck_nirspec_high, keck_nirspec_high_old, + providing files directly: aat_uhrf, bok_bc, + gemini_flamingos1, gemini_flamingos2, + gemini_gmos_north_e2v, gemini_gmos_north_ham, + gemini_gmos_north_ham_ns, gemini_gmos_south_ham, + gemini_gnirs_echelle, gemini_gnirs_ifu, gtc_maat, + gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_high, keck_nirspec_high_old, keck_nirspec_low, lbt_luci1, lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, ldt_deveny, magellan_fire, magellan_fire_long, magellan_mage, diff --git a/doc/help/pypeit_view_fits.rst b/doc/help/pypeit_view_fits.rst index fb6bbd14e2..4ee223de61 100644 --- a/doc/help/pypeit_view_fits.rst +++ b/doc/help/pypeit_view_fits.rst @@ -9,7 +9,7 @@ View FITS files with ginga positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index a0d772877b..4cb290e4fa 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -7,11 +7,11 @@ ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev336+gdf3013372.d20240827 ## ## Available spectrographs include: - ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, - ## gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - ## gemini_gmos_south_ham, gemini_gnirs_echelle, gemini_gnirs_ifu, - ## gtc_maat, gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, - ## keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + ## aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, + ## gemini_gmos_north_e2v, gemini_gmos_north_ham, + ## gemini_gmos_north_ham_ns, gemini_gmos_south_ham, gemini_gnirs_echelle, + ## gemini_gnirs_ifu, gtc_maat, gtc_osiris, gtc_osiris_plus, jwst_nircam, + ## jwst_nirspec, keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, ## keck_lris_blue, keck_lris_blue_orig, keck_lris_red, ## keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, keck_nires, ## keck_nirspec_high, keck_nirspec_high_old, keck_nirspec_low, lbt_luci1, diff --git a/doc/include/inst_detector_table.rst b/doc/include/inst_detector_table.rst index 3aa03054fc..41c4c8b132 100644 --- a/doc/include/inst_detector_table.rst +++ b/doc/include/inst_detector_table.rst @@ -1,6 +1,7 @@ ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== Instrument Det specaxis specflip spatflip namp gain RN darkcurr min sat nonlinear platescale ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== +``aat_uhrf`` 1 0 False False 1 1.0 0.0 0.0 -1.0e+10 65535.0 0.7600 0.0500 ``bok_bc`` 1 1 False False 1 1.5 3.0 5.4 -1.0e+10 65535.0 1.0000 0.2000 ``gemini_flamingos1`` 1 0 False False 1 3.8 6.0 1080.0 -1.0e+10 320000.0 0.8750 0.1500 ``gemini_flamingos2`` 1 0 True False 1 4.44 5.0 1800.0 -1.0e+10 700000.0 1.0000 0.1787 diff --git a/doc/include/spectrographs_table.rst b/doc/include/spectrographs_table.rst index a0705892d3..04b73891e7 100644 --- a/doc/include/spectrographs_table.rst +++ b/doc/include/spectrographs_table.rst @@ -1,6 +1,7 @@ ======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== ``PypeIt`` Name ``PypeIt`` Class Telescope Camera URL Pipeline Supported QL Tested Comments ======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== +aat_uhrf :class:`~pypeit.spectrographs.aat_uhrf.AATUHRFSpectrograph` AAT UHRF `Link `__ MultiSlit True False bok_bc :class:`~pypeit.spectrographs.bok_bc.BokBCSpectrograph` BOK BC `Link `__ MultiSlit True False Bok B&C spectrometer gemini_flamingos1 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS1Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit False False gemini_flamingos2 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS2Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit True False Flamingos-2 NIR spectrograph diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index 1851d9f78e..4cd22c89e9 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -893,7 +893,6 @@ Key Type Opti ``overscan_par`` int, list .. 5, 65 Parameters for the overscan subtraction. For 'chebyshev' or 'polynomial', set overcan_par = order; for 'savgol', set overscan_par = order, window size ; for 'median', set overscan_par = None or omit the keyword. ``rmcompact`` bool .. True Remove compact detections in LA cosmics routine ``satpix`` str ``reject``, ``force``, ``nothing`` ``reject`` Handling of saturated pixels. Options are: reject, force, nothing -``scale_to_mean`` bool .. False If True, scale the input images to have the same mean before combining. ``scattlight`` :class:`~pypeit.par.pypeitpar.ScatteredLightPar` .. `ScatteredLightPar Keywords`_ Scattered light subtraction parameters. ``shot_noise`` bool .. True Use the bias- and dark-subtracted image to calculate and include electron count shot noise in the image processing error budget ``sigclip`` int, float .. 4.5 Sigma level for rejection in LA cosmics routine @@ -1058,6 +1057,98 @@ these in the PypeIt file, you would be reproducing the effect of the `default_pypeit_par` method specific to each derived :class:`~pypeit.spectrographs.spectrograph.Spectrograph` class. +.. _instr_par-aat_uhrf: + +AAT UHRF (``aat_uhrf``) +----------------------- +Alterations to the default parameters are: + +.. code-block:: ini + + [rdx] + spectrograph = aat_uhrf + [calibrations] + [[biasframe]] + [[[process]]] + combine = median + use_biasimage = False + shot_noise = False + use_pixelflat = False + use_illumflat = False + [[darkframe]] + [[[process]]] + mask_cr = True + use_pixelflat = False + use_illumflat = False + [[arcframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[tiltframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[pixelflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[alignframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[traceframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[illumflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[lampoffflatsframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[scattlightframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[skyframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[standardframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[wavelengths]] + lamps = ThAr, + n_final = 3 + [[slitedges]] + sync_predict = nearest + bound_detector = True + [[tilts]] + spat_order = 4 + spec_order = 1 + [scienceframe] + exprng = 61, None, + [[process]] + mask_cr = True + sigclip = 10.0 + noise_floor = 0.01 + [reduce] + [[skysub]] + bspline_spacing = 3.0 + no_poly = True + user_regions = :10,75: + .. _instr_par-bok_bc: BOK BC (``bok_bc``) diff --git a/doc/releases/1.16.1dev.rst b/doc/releases/1.16.1dev.rst index 6b5ee0e5f1..6380d44414 100644 --- a/doc/releases/1.16.1dev.rst +++ b/doc/releases/1.16.1dev.rst @@ -45,6 +45,10 @@ Instrument-specific Updates - Improved LRIS frame typing, including the typing of slitless flats and sky flats. - Improved HIRES frame typing and configuration setup. +- Added support for the (decommissioned) AAT/UHRF instrument +- Updated the requirements of a spectral flip for KCWI (blue) data. If all amplifiers are + used, the data will be flipped. If all amplifiers are used, the data will not be flipped. + Otherwise, the data will be flipped. - Updated X-Shooter detector gain and read noise to come from header, and updated plate scales to the most recent values from the manual. Detailed changes are: @@ -88,6 +92,8 @@ Script Changes generated during the reduction and stored in ``data/pixelflats``. - Added ``pypeit_chk_flexure`` script to check both spatial and spectral flexure applied to the reduced data. +- Treatment of file names is now more formal. Compression signatures are now + considered, and filename matching is now more strict. Datamodel Changes ----------------- diff --git a/doc/setup.rst b/doc/setup.rst index 2a046b17e5..523ed2290d 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -22,6 +22,27 @@ preparatory script, :ref:`pypeit_obslog`, which provides a simple listing of the available data files; however, use of this script is optional in terms of setup for reducing your data. +.. _setup-file-searching: + +Raw File Searches +================= + +PypeIt scripts that search for raw files in a given directory base the search on +a list of file extensions. These are generally ``.fits`` and ``.fits.gz``, but +some spectrographs specify a different set. + +Some scripts allow you to specify the extension to use for the search, which +*must* be one of the allowed extensions for that spectrograph. E.g., +for a spectrograph that allows ``.fits`` and ``.fits.gz`` extension, you can +specify to only look for the ``.fits`` files, but you *cannot* have it look for +``.fits.bz2`` files. If your raw files have extensions that are currently not +allowed by the code, please `Submit an issue`_. + +If you have both compressed and uncompressed files in your directory, the search +function will generally find both. You are strongly encouraged to only include +one version (compressed or uncompressed) of each file in the directory with your +raw data. + .. _setup-metadata: Use of Metadata to Identify Instrument Configurations @@ -168,7 +189,9 @@ to be the same directory that holds the raw data. 1. First Execution ------------------ -We recommend you first execute ``pypeit_setup`` like this:: +We recommend you first execute ``pypeit_setup`` like this: + +.. code-block:: bash pypeit_setup -r path_to_your_raw_data/LB -s keck_lris_blue @@ -187,7 +210,9 @@ This execution of ``pypeit_setup`` searches for all `*.fits` and `*.fits.gz` files with the provided root directory. Generally, the provided path should **not** contain a wild-card and it is best if you provide the *full* path; however, you can search through multiple -directories as follows:: +directories as follows: + +.. code-block:: bash pypeit_setup -r "/Users/xavier/Keck/LRIS/data/2016apr06/Raw/*/LB" -s keck_lris_blue diff --git a/doc/spectrographs/aat_uhrf.rst b/doc/spectrographs/aat_uhrf.rst new file mode 100644 index 0000000000..ebef209ecf --- /dev/null +++ b/doc/spectrographs/aat_uhrf.rst @@ -0,0 +1,33 @@ +.. highlight:: rest + +******** +AAT UHRF +******** + + +Overview +======== + +This file summarizes several instrument specific +settings that are related to AAT/UHRF. + + +Wavelength Calibration +---------------------- + +UHRF has many wavelength setups, and the wavelength calibration +must be performed manually for each setup using :ref:`wvcalib-byhand` +approach and the :ref:`pypeit_identify` script. Since this spectrograph +is decommissioned, we do not expect to have a general solution +for this spectrograph. + +Object profile +-------------- + +UHRF is a slicer spectrograph, and the data are usually binned aggressively. +The object profile tends to be poorly characterised with the automated approach, +and you may need to generate your own wavelength dependent profile. Previously, +a Gaussian KDE profile was used, and this performed well, but is not available +by default. For users that are interested in this functionality, please contact +the PypeIt developers on the PypeIt User's Slack, or see the `aat_uhrf_rjc` +branch. diff --git a/doc/spectrographs/spectrographs.rst b/doc/spectrographs/spectrographs.rst index 275915a2c9..1fffa34730 100644 --- a/doc/spectrographs/spectrographs.rst +++ b/doc/spectrographs/spectrographs.rst @@ -33,6 +33,7 @@ instrument-specific details for running PypeIt. :caption: Spectrographs :maxdepth: 1 + aat_uhrf gemini_flamingos2 gemini_gmos gemini_gnirs diff --git a/presentations/py/users.py b/presentations/py/users.py index 9f3affcddd..73902997e4 100644 --- a/presentations/py/users.py +++ b/presentations/py/users.py @@ -21,9 +21,9 @@ def set_fontsize(ax, fsz): ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(fsz) -user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10"] +user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10", "2024-08-20"] user_dates = numpy.array([numpy.datetime64(date) for date in user_dates]) -user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620]) +user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620, 643]) user_pred_dates = numpy.array([numpy.datetime64(date) for date in ["2024-06-10", "2024-12-31", "2025-12-31", "2026-12-31", diff --git a/pypeit/core/arc.py b/pypeit/core/arc.py index 086f7deb04..e41fc7fc8b 100644 --- a/pypeit/core/arc.py +++ b/pypeit/core/arc.py @@ -507,6 +507,7 @@ def get_censpec(slit_cen, slitmask, arcimg, gpm=None, box_rad=3.0, arc_spec[arc_spec_bpm] = 0.0 return arc_spec, arc_spec_bpm, np.all(arc_spec_bpm, axis=0) + def detect_peaks(x, mph=None, mpd=1, threshold=0, edge='rising', kpsh=False, valley=False, show=False, ax=None): """Detect peaks in data based on their amplitude and other features. @@ -783,7 +784,7 @@ def iter_continuum(spec, gpm=None, fwhm=4.0, sigthresh = 2.0, sigrej=3.0, niter_ max_nmask = int(np.ceil((max_mask_frac)*nspec_available)) for iter in range(niter_cont): spec_sub = spec - cont_now - mask_sigclip = np.invert(cont_mask & gpm) + mask_sigclip = np.logical_not(cont_mask & gpm) (mean, med, stddev) = stats.sigma_clipped_stats(spec_sub, mask=mask_sigclip, sigma_lower=sigrej, sigma_upper=sigrej, cenfunc='median', stdfunc=utils.nan_mad_std) # be very liberal in determining threshold for continuum determination diff --git a/pypeit/core/findobj_skymask.py b/pypeit/core/findobj_skymask.py index f919f70bda..6e4698c844 100644 --- a/pypeit/core/findobj_skymask.py +++ b/pypeit/core/findobj_skymask.py @@ -1764,8 +1764,7 @@ def objs_in_slit(image, ivar, thismask, slit_left, slit_righ, flux_sum_smash = np.sum((image_rect*gpm_sigclip)[find_min_max_out[0]:find_min_max_out[1]], axis=0) flux_smash = flux_sum_smash*gpm_smash/(npix_smash + (npix_smash == 0.0)) flux_smash_mean, flux_smash_med, flux_smash_std = astropy.stats.sigma_clipped_stats( - flux_smash, mask=np.logical_not(gpm_smash), sigma_lower=3.0, sigma_upper=3.0 - ) + flux_smash, mask=np.logical_not(gpm_smash), sigma_lower=3.0, sigma_upper=3.0) flux_smash_recen = flux_smash - flux_smash_med # Return if none found and no hand extraction diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index 6c0d333a59..b4d5a7abf7 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -248,7 +248,6 @@ def global_skysub(image, ivar, tilts, thismask, slit_left, slit_righ, inmask=Non return ythis - def skyoptimal(piximg, data, ivar, oprof, sigrej=3.0, npoly=1, spatial_img=None, fullbkpt=None): """ Utility routine used by local_skysub_extract that performs the joint b-spline fit for sky-background @@ -915,7 +914,7 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, isub, = np.where(localmask.flatten()) #sortpix = (piximg.flat[isub]).argsort() obj_profiles_flat = obj_profiles.reshape(nspec * nspat, objwork) - skymask = outmask & np.invert(edgmask) + skymask = outmask & np.logical_not(edgmask) sky_bmodel, obj_bmodel, outmask_opt = skyoptimal( piximg.flat[isub], sciimg.flat[isub], (modelivar * skymask).flat[isub], obj_profiles_flat[isub, :], spatial_img=spatial_img.flat[isub], diff --git a/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits b/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits new file mode 100644 index 0000000000..4e2009a902 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits differ diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index e2e5fd10ce..b7d5868ee8 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -1650,12 +1650,15 @@ def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', profile. Shape is :math:`(N_{\rm flat},)`. method (:obj:`str`, optional): Method to use for tweaking the slit edges. Options are: - - 'threshold': Use the threshold to set the slit edge + + - ``'threshold'``: Use the threshold to set the slit edge and then shift it to the left or right based on the illumination profile. - - 'gradient': Use the gradient of the illumination - profile to set the slit edge and then shift it to - the left or right based on the illumination profile. + + - ``'gradient'``: Use the gradient of the illumination + profile to set the slit edge and then shift it to the left + or right based on the illumination profile. + thresh (:obj:`float`, optional): Threshold of the normalized flat profile at which to place the two slit edges. @@ -1674,12 +1677,12 @@ def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', tuple: Returns six objects: - The threshold used to set the left edge - - The fraction of the slit that the left edge is shifted to - the right + - The fraction of the slit that the left edge is shifted to the + right - The adjusted left edge - The threshold used to set the right edge - - The fraction of the slit that the right edge is shifted to - the left + - The fraction of the slit that the right edge is shifted to the + left - The adjusted right edge """ diff --git a/pypeit/images/combineimage.py b/pypeit/images/combineimage.py index 3360d731c1..31980b00b5 100644 --- a/pypeit/images/combineimage.py +++ b/pypeit/images/combineimage.py @@ -25,7 +25,7 @@ class CombineImage: rawImages (:obj:`list`, :class:`~pypeit.images.pypeitimage.PypeItImage`): Either a single :class:`~pypeit.images.pypeitimage.PypeItImage` object or a list of one or more of these objects to be combined into - a an image. + an image. par (:class:`~pypeit.par.pypeitpar.ProcessImagesPar`): Parameters that dictate the processing of the images. diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 952f40142b..52a41f7c11 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -1149,6 +1149,5 @@ def grab_rawfiles(file_of_files:str=None, list_of_files:list=None, raw_paths:lis return [str(p / f) for p in _raw_paths for f in list_of_files if (p / f).exists()] # Find all files that have the correct extension - return np.concatenate([files_from_extension(str(p), extension=extension) - for p in _raw_paths]).tolist() + return files_from_extension(_raw_paths, extension=extension) diff --git a/pypeit/io.py b/pypeit/io.py index 2e0c13b992..1d230ca5eb 100644 --- a/pypeit/io.py +++ b/pypeit/io.py @@ -851,40 +851,48 @@ def create_symlink(filename, symlink_dir, relative_symlink=False, overwrite=Fals os.symlink(olink_src, olink_dest) -def files_from_extension(raw_path, - extension:str='fits'): +def files_from_extension(raw_path, extension='.fits'): """ - Grab the list of files with a given extension + Find files from one or more paths with one or more extensions. - Args: - raw_path (str or list): - Path(s) to raw files, which may or may not include the prefix of the - files to search for. - - For a string input, for example, this can be the directory - ``'/path/to/files/'`` or the directory plus the file prefix - ``'/path/to/files/prefix'``, which yeilds the search strings - ``'/path/to/files/*fits'`` or ``'/path/to/files/prefix*fits'``, - respectively. - - For a list input, this can use wildcards for multiple directories. + This is a recursive function. If ``raw_path`` is a list, the function is + called for every item in the list and the results are concatenated. - extension (str, optional): - File extension to search on. + Args: + raw_path (:obj:`str`, `Path`_, :obj:`list`): + One or more paths to search for files, which may or may not include + the prefix of the files to search for. For string input, this can + be the directory ``'/path/to/files/'`` or the directory plus the + file prefix ``'/path/to/files/prefix'``, which yeilds the search + strings ``'/path/to/files/*fits'`` or + ``'/path/to/files/prefix*fits'``, respectively. For a list input, + this can use wildcards for multiple directories. + + extension (:obj:`str`, :obj:`list`, optional): + One or more file extensions to search on. Returns: - list: List of raw data filenames (sorted) with full path - """ - if isinstance(raw_path, str): - # Grab the list of files - dfname = os.path.join(raw_path, f'*{extension}*') \ - if os.path.isdir(raw_path) else f'{raw_path}*{extension}*' - return sorted(glob.glob(dfname)) + :obj:`list`: List of `Path`_ objects with the full path to the set of + unique raw data filenames that match the provided criteria search + strings. + """ + if isinstance(raw_path, (str, Path)): + _raw_path = Path(raw_path).absolute() + if _raw_path.is_dir(): + prefix = '' + else: + _raw_path, prefix = _raw_path.parent, _raw_path.name + if not _raw_path.is_dir(): + msgs.error(f'{_raw_path} does not exist!') + ext = [extension] if isinstance(extension, str) else extension + files = numpy.concatenate([sorted(_raw_path.glob(f'{prefix}*{e}')) for e in ext]) + return numpy.unique(files).tolist() if isinstance(raw_path, list): - return numpy.concatenate([files_from_extension(p, extension=extension) for p in raw_path]).tolist() + files = numpy.concatenate([files_from_extension(p, extension=extension) for p in raw_path]) + return numpy.unique(files).tolist() - msgs.error(f"Incorrect type {type(raw_path)} for raw_path (must be str or list)") + msgs.error(f"Incorrect type {type(raw_path)} for raw_path; must be str, Path, or list.") diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index 4f413a0673..4f7498c169 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -5331,7 +5331,7 @@ def valid_telescopes(): """ Return the valid telescopes. """ - return [ 'GEMINI-N','GEMINI-S', 'KECK', 'SHANE', 'WHT', 'APF', 'TNG', 'VLT', 'MAGELLAN', 'LBT', 'MMT', + return ['AAT', 'GEMINI-N','GEMINI-S', 'KECK', 'SHANE', 'WHT', 'APF', 'TNG', 'VLT', 'MAGELLAN', 'LBT', 'MMT', 'KPNO', 'NOT', 'P200', 'BOK', 'GTC', 'SOAR', 'NTT', 'LDT', 'JWST', 'HILTNER'] def validate(self): diff --git a/pypeit/pypeitsetup.py b/pypeit/pypeitsetup.py index 74a82159dc..20b95b8c03 100644 --- a/pypeit/pypeitsetup.py +++ b/pypeit/pypeitsetup.py @@ -163,30 +163,40 @@ def from_pypeit_file(cls, filename): cfg_lines=pypeItFile.cfg_lines, pypeit_file=filename) - # TODO: Make the default here match the default used by - # io.files_from_extension? @classmethod - def from_file_root(cls, root, spectrograph, extension='.fits'): + def from_file_root(cls, root, spectrograph, extension=None): """ Instantiate the :class:`~pypeit.pypeitsetup.PypeItSetup` object by providing a file root. Args: - root (:obj:`str`): - String used to find the raw files; see + root (:obj:`str`, `Path`_, :obj:`list`): + One or more paths within which to search for files; see :func:`~pypeit.io.files_from_extension`. - spectrograph (:obj:`str`): + spectrograph (:obj:`str`, :class:`~pypeit.spectrographs.spectrograph.Spectrograph`): The PypeIt name of the spectrograph used to take the observations. This should be one of the available options in :attr:`~pypeit.spectrographs.available_spectrographs`. - extension (:obj:`str`, optional): + extension (:obj:`str`, :obj:`list`, optional): The extension common to all the fits files to reduce; see - :func:`~pypeit.io.files_from_extension`. - + :func:`~pypeit.io.files_from_extension`. If None, uses the + ``allowed_extensions`` of the spectrograph class. Otherwise, + this *must* be a subset of the allowed extensions for the + selected spectrograph. + Returns: :class:`PypeitSetup`: The instance of the class. """ - return cls.from_rawfiles(io.files_from_extension(root, extension=extension), spectrograph) + # NOTE: This works if `spectrograph` is either a string or a + # Spectrograph object + spec = load_spectrograph(spectrograph).__class__ + files = spec.find_raw_files(root, extension=extension) + nfiles = len(files) + if nfiles == 0: + msgs.error(f'Unable to find any raw files for {spec.name} in {root}!') + else: + msgs.info(f'Found {nfiles} {spec.name} raw files.') + return cls.from_rawfiles(files, spectrograph) @classmethod def from_rawfiles(cls, data_files:list, spectrograph:str, frametype=None): diff --git a/pypeit/scripts/chk_for_calibs.py b/pypeit/scripts/chk_for_calibs.py index ed3000d992..9269529fa7 100644 --- a/pypeit/scripts/chk_for_calibs.py +++ b/pypeit/scripts/chk_for_calibs.py @@ -21,8 +21,11 @@ def get_parser(cls, width=None): parser.add_argument('-s', '--spectrograph', default=None, type=str, help='A valid spectrograph identifier: {0}'.format( ', '.join(available_spectrographs))) - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('--save_setups', default=False, action='store_true', help='If not toggled, remove setup_files/ folder and its files.') return parser diff --git a/pypeit/scripts/coadd_1dspec.py b/pypeit/scripts/coadd_1dspec.py index 7763f9b905..98306e1126 100644 --- a/pypeit/scripts/coadd_1dspec.py +++ b/pypeit/scripts/coadd_1dspec.py @@ -22,69 +22,6 @@ from pypeit.spectrographs.util import load_spectrograph -## TODO: This is basically the exact same code as read_fluxfile in the fluxing -## script. Consolidate them? Make this a standard method in parse or io. -#def read_coaddfile(ifile): -# """ -# Read a ``PypeIt`` coadd1d file, akin to a standard PypeIt file. -# -# The top is a config block that sets ParSet parameters. The name of the -# spectrograph is required. -# -# Args: -# ifile (:obj:`str`): -# Name of the coadd file -# -# Returns: -# :obj:`tuple`: Three objects are returned: a :obj:`list` with the -# configuration entries used to modify the relevant -# :class:`~pypeit.par.parset.ParSet` parameters, a :obj:`list` the names -# of spec1d files to be coadded, and a :obj:`list` with the object IDs -# aligned with each of the spec1d files. -# """ -# # Read in the pypeit reduction file -# msgs.info('Loading the coadd1d file') -# lines = inputfiles.read_pypeit_file_lines(ifile) -# is_config = np.ones(len(lines), dtype=bool) -# -# -# # Parse the fluxing block -# spec1dfiles = [] -# objids_in = [] -# s, e = inputfiles.InputFile.find_block(lines, 'coadd1d') -# if s >= 0 and e < 0: -# msgs.error("Missing 'coadd1d end' in {0}".format(ifile)) -# elif (s < 0) or (s==e): -# msgs.error("Missing coadd1d read or [coadd1d] block in in {0}. Check the input format for the .coadd1d file".format(ifile)) -# else: -# for ctr, line in enumerate(lines[s:e]): -# prs = line.split(' ') -# spec1dfiles.append(prs[0]) -# if ctr == 0 and len(prs) != 2: -# msgs.error('Invalid format for .coadd1d file.' + msgs.newline() + -# 'You must have specify a spec1dfile and objid on the first line of the coadd1d block') -# if len(prs) > 1: -# objids_in.append(prs[1]) -# is_config[s-1:e+1] = False -# -# # Chck the sizes of the inputs -# nspec = len(spec1dfiles) -# if len(objids_in) == 1: -# objids = nspec*objids_in -# elif len(objids_in) == nspec: -# objids = objids_in -# else: -# msgs.error('Invalid format for .flux file.' + msgs.newline() + -# 'You must specify a single objid on the first line of the coadd1d block,' + msgs.newline() + -# 'or specify am objid for every spec1dfile in the coadd1d block.' + msgs.newline() + -# 'Run pypeit_coadd_1dspec --help for information on the format') -# # Construct config to get spectrograph -# cfg_lines = list(lines[is_config]) -# -# # Return -# return cfg_lines, spec1dfiles, objids - - def build_coadd_file_name(spec1dfiles, spectrograph): """Build the output file name for coadding. The filename convention is coadd1d___.fits or @@ -116,6 +53,7 @@ def build_coadd_file_name(spec1dfiles, spectrograph): path = os.path.dirname(os.path.abspath(spec1dfiles[0])) return os.path.join(path, f'coadd1d_{target}_{instrument_name}_{date_portion}.fits') + class CoAdd1DSpec(scriptbase.ScriptBase): @classmethod diff --git a/pypeit/scripts/multislit_flexure.py b/pypeit/scripts/multislit_flexure.py index 98e836196b..1691361687 100644 --- a/pypeit/scripts/multislit_flexure.py +++ b/pypeit/scripts/multislit_flexure.py @@ -18,55 +18,6 @@ from pypeit.scripts import scriptbase -#def read_flexfile(ifile): -# """ -# Read a ``PypeIt`` flexure file, akin to a standard ``PypeIt`` file. -# -# The top is a config block that sets ParSet parameters. -# -# Args: -# ifile (:obj:`str`): -# Name of the flexure file -# -# Returns: -# :obj:`tuple`: Two objects are returned: a :obj:`list` with the -# configuration entries used to modify the relevant -# :class:`~pypeit.par.parset.ParSet` parameters and a :obj:`list` with the -# names of spec1d files to be flexure corrected. -# """ -# # Read in the pypeit reduction file -# msgs.info('Loading the flexure file') -# lines = inputfiles.read_pypeit_file_lines(ifile) -# is_config = np.ones(len(lines), dtype=bool) -# -# # Parse the fluxing block -# spec1dfiles = [] -# objids_in = [] -# s, e = inputfiles.InputFile.find_block(lines, 'flexure') -# if s >= 0 and e < 0: -# msgs.error("Missing 'flexure end' in {0}".format(ifile)) -# elif (s < 0) or (s == e): -# msgs.error( -# "Missing flexure read block in {0}. Check the input format for the .flex file".format(ifile)) -# else: -# for ctr, line in enumerate(lines[s:e]): -# prs = line.split(' ') -# spec1dfiles.append(prs[0]) -# if len(prs) > 1: -# msgs.error('Invalid format for .flex file.' + msgs.newline() + -# 'You must specify only spec1dfiles in the block ') -# is_config[s-1:e+1] = False -# -# # Chck the sizes of the inputs -# nspec = len(spec1dfiles) -# -# # Construct config to get spectrograph -# cfg_lines = list(lines[is_config]) -# -# # Return -# return cfg_lines, spec1dfiles - - # TODO: Maybe not a good idea to name this script the same as the # flexure.MultiSlitFlexure class, but it is technically okay... class MultiSlitFlexure(scriptbase.ScriptBase): diff --git a/pypeit/scripts/obslog.py b/pypeit/scripts/obslog.py index 693b670a0c..6e1e7a7657 100644 --- a/pypeit/scripts/obslog.py +++ b/pypeit/scripts/obslog.py @@ -65,8 +65,11 @@ def get_parser(cls, width=None): parser.add_argument('-s', '--sort', default='mjd', type=str, help='Metadata keyword (pypeit-specific) to use to sort the output ' 'table.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-d', '--output_path', default='current working directory', help='Path to top-level output directory.') parser.add_argument('-o', '--overwrite', default=False, action='store_true', @@ -115,8 +118,7 @@ def main(args): f'argument.') # Generate the metadata table - ps = PypeItSetup.from_file_root(args.root, args.spec, - extension=args.extension) + ps = PypeItSetup.from_file_root(args.root, args.spec, extension=args.extension) ps.run(setup_only=True, # This allows for bad headers groupings=args.groupings, clean_config=args.bad_frames) diff --git a/pypeit/scripts/setup.py b/pypeit/scripts/setup.py index 9de645baa3..f417caf172 100644 --- a/pypeit/scripts/setup.py +++ b/pypeit/scripts/setup.py @@ -28,8 +28,11 @@ def get_parser(cls, width=None): 'directory (e.g., /data/Kast) or the search string up through ' 'the wildcard (.e.g, /data/Kast/b). Use the --extension option ' 'to set the types of files to search for.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-d', '--output_path', default='current working directory', help='Path to top-level output directory.') parser.add_argument('-o', '--overwrite', default=False, action='store_true', diff --git a/pypeit/scripts/setup_gui.py b/pypeit/scripts/setup_gui.py index d864eb0372..b8925be7e6 100644 --- a/pypeit/scripts/setup_gui.py +++ b/pypeit/scripts/setup_gui.py @@ -27,8 +27,11 @@ def get_parser(cls, width=None): 'the wildcard (.e.g, /data/Kast/b). Use the --extension option ' 'to set the types of files to search for. Default is the ' 'current working directory.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-l', '--logfile', type=str, default=None, help="Write the PypeIt logs to the given file. If the file exists it will be renamed.") parser.add_argument('-v', '--verbosity', type=int, default=2, diff --git a/pypeit/spectrographs/__init__.py b/pypeit/spectrographs/__init__.py index 3377156931..06efedfb57 100644 --- a/pypeit/spectrographs/__init__.py +++ b/pypeit/spectrographs/__init__.py @@ -3,11 +3,12 @@ # The import of all the spectrograph modules here is what enables the dynamic # compiling of all the available spectrographs below -from pypeit.spectrographs import gtc_osiris +from pypeit.spectrographs import aat_uhrf from pypeit.spectrographs import bok_bc from pypeit.spectrographs import gemini_flamingos from pypeit.spectrographs import gemini_gmos from pypeit.spectrographs import gemini_gnirs +from pypeit.spectrographs import gtc_osiris from pypeit.spectrographs import keck_esi from pypeit.spectrographs import keck_deimos from pypeit.spectrographs import keck_hires diff --git a/pypeit/spectrographs/aat_uhrf.py b/pypeit/spectrographs/aat_uhrf.py new file mode 100644 index 0000000000..ea71c1ed10 --- /dev/null +++ b/pypeit/spectrographs/aat_uhrf.py @@ -0,0 +1,257 @@ +""" +Module for Shane/Kast specific methods. + +.. include:: ../include/links.rst +""" +import os + +from IPython import embed + +import numpy as np + +from astropy.time import Time + +from pypeit import msgs +from pypeit import telescopes +from pypeit.core import framematch +from pypeit.spectrographs import spectrograph +from pypeit.images import detector_container +from pypeit import data + + +class AATUHRFSpectrograph(spectrograph.Spectrograph): + """ + Child to handle AAT/UHRF specific code + """ + ndet = 1 + telescope = telescopes.AATTelescopePar() + url = 'https://aat.anu.edu.au/science/instruments/decomissioned/uhrf/overview' + ql_supported = False + name = 'aat_uhrf' + camera = 'UHRF' + supported = True + header_name = 'uhrf' + allowed_extensions = [".FTS"] + + def get_detector_par(self, det, hdu=None): + """ + Return metadata for the selected detector. + + Args: + det (:obj:`int`): + 1-indexed detector number. + hdu (`astropy.io.fits.HDUList`_, optional): + The open fits file with the raw image of interest. If not + provided, frame-dependent parameters are set to a default. + + Returns: + :class:`~pypeit.images.detector_container.DetectorContainer`: + Object with the detector metadata. + """ + # Retrieve the binning + binning = '1,1' if hdu is None else self.compound_meta(self.get_headarr(hdu), "binning") + dsec = 1 + 1024//int(binning.split(',')[0]) + # Detector 1 + detector_dict = dict( + binning=binning, + det=1, + dataext=0, + specaxis=0, + specflip=False, + spatflip=False, + platescale=0.05, # Not sure about this value + saturation=65535., + mincounts=-1e10, + nonlinear=0.76, + numamplifiers=1, + gain=np.asarray([1.0]), # Not sure about this value + ronoise=np.asarray([0.0]), # Determine the read noise from the overscan region + xgap=0., + ygap=0., + ysize=1., + darkcurr=0.0, # e-/pixel/hour + # These are rows, columns on the raw frame, 1-indexed + datasec=np.asarray(['[:, 1:{:d}]'.format(dsec)]), + oscansec=np.asarray(['[:, {:d}:]'.format(dsec+1)]) + ) + return detector_container.DetectorContainer(**detector_dict) + + @classmethod + def default_pypeit_par(cls): + """ + Return the default parameters to use for this instrument. + + Returns: + :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by + all of PypeIt methods. + """ + par = super().default_pypeit_par() + + # Ignore PCA + par['calibrations']['slitedges']['sync_predict'] = 'nearest' + # Bound the detector with slit edges if no edges are found + par['calibrations']['slitedges']['bound_detector'] = True + + # Never correct for flexure - the sky is subdominant compared to the object and basically never detected. + par['flexure']['spec_method'] = 'skip' + + # Sky subtraction parameters - this instrument has no sky lines, but we still use the sky subtraction + # routine to subtract scattered light. + par['reduce']['skysub']['no_poly'] = True + par['reduce']['skysub']['bspline_spacing'] = 3.0 + par['reduce']['skysub']['user_regions'] = ':10,75:' # This is about right for most setups tested so far + par['scienceframe']['process']['sigclip'] = 10.0 + + # Set some parameters for the calibrations + # par['calibrations']['wavelengths']['reid_arxiv'] = 'None' + par['calibrations']['wavelengths']['lamps'] = ['ThAr'] + par['calibrations']['wavelengths']['n_final'] = 3 + par['calibrations']['tilts']['spat_order'] = 4 + par['calibrations']['tilts']['spec_order'] = 1 + + # Set the default exposure time ranges for the frame typing + # Trace frames should be the same as arc frames - it will force a bound detector and this + # allows the scattered light to be subtracted. A pixel-to-pixel sensitivity correction is + # not needed for this instrument, since it's a small slicer that projects the target onto + # multiple pixels. This instrument observes bright objects only, so sky subtraction is not + # important, but the sky subtraction routine is used to subtract scattered light, instead. + par['calibrations']['arcframe']['exprng'] = [None, 60.0] + par['calibrations']['tiltframe']['exprng'] = [None, 60.0] + par['calibrations']['traceframe']['exprng'] = [None, 60.0] + par['scienceframe']['exprng'] = [61, None] + + return par + + def init_meta(self): + """ + Define how metadata are derived from the spectrograph files. + + That is, this associates the PypeIt-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + self.meta = {} + # Required (core) + self.meta['ra'] = dict(ext=0, card='MEANRA') + self.meta['dec'] = dict(ext=0, card='MEANDEC') + self.meta['target'] = dict(ext=0, card='OBJECT') + # dispname is arm specific (blue/red) + self.meta['decker'] = dict(ext=0, card='WINDOW') + self.meta['dispname'] = dict(ext=0, card='WINDOW') + self.meta['binning'] = dict(ext=0, card=None, compound=True) + self.meta['mjd'] = dict(ext=0, card=None, compound=True) + self.meta['exptime'] = dict(ext=0, card='TOTALEXP') + self.meta['airmass'] = dict(ext=0, card=None, compound=True) + # Additional ones, generally for configuration determination or time + # self.meta['dichroic'] = dict(ext=0, card='BSPLIT_N') + # self.meta['instrument'] = dict(ext=0, card='VERSION') + + def compound_meta(self, headarr, meta_key): + """ + Methods to generate metadata requiring interpretation of the header + data, instead of simply reading the value of a header card. + + Args: + headarr (:obj:`list`): + List of `astropy.io.fits.Header`_ objects. + meta_key (:obj:`str`): + Metadata keyword to construct. + + Returns: + object: Metadata value read from the header(s). + """ + if meta_key == 'mjd': + date = headarr[0]['UTDATE'].replace(":","-") + time = headarr[0]['UTSTART'] + ttime = Time(f'{date}T{time}', format='isot') + return ttime.mjd + elif meta_key == 'binning': + binspat = int(np.ceil(1024/headarr[0]['NAXIS1'])) + binspec = int(np.ceil(1024/headarr[0]['NAXIS2'])) + return f'{binspat},{binspec}' + elif meta_key == 'airmass': + # Calculate the zenith distance + zendist = 0.5*(headarr[0]['ZDSTART']+headarr[0]['ZDEND']) + # Return the airmass based on the zenith distance + return 1./np.cos(np.deg2rad(zendist)) + msgs.error("Not ready for this compound meta") + + def configuration_keys(self): + """ + Return the metadata keys that define a unique instrument + configuration. + + This list is used by :class:`~pypeit.metadata.PypeItMetaData` to + identify the unique configurations among the list of frames read + for a given reduction. + + Returns: + :obj:`list`: List of keywords of data pulled from file headers + and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + object. + """ + # decker is not included because arcs are often taken with a 0.5" slit + return ['dispname'] + + def check_frame_type(self, ftype, fitstbl, exprng=None): + """ + Check for frames of the provided type. + + Args: + ftype (:obj:`str`): + Type of frame to check. Must be a valid frame type; see + frame-type :ref:`frame_type_defs`. + fitstbl (`astropy.table.Table`_): + The table with the metadata for one or more frames to check. + exprng (:obj:`list`, optional): + Range in the allowed exposure time for a frame of type + ``ftype``. See + :func:`pypeit.core.framematch.check_frame_exptime`. + + Returns: + `numpy.ndarray`_: Boolean array with the flags selecting the + exposures in ``fitstbl`` that are ``ftype`` type frames. + """ + good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) + if ftype in ['science']: + return good_exp + if ftype in ['standard']: + return np.zeros(len(fitstbl), dtype=bool) + if ftype == 'bias': + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['pixelflat', 'trace', 'illumflat']: + # Flats and trace frames are typed together + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['pinhole', 'dark']: + # Don't type pinhole or dark frames + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['arc', 'tilt']: + return good_exp + + msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) + return np.zeros(len(fitstbl), dtype=bool) + + def config_specific_par(self, scifile, inp_par=None): + """ + Modify the PypeIt parameters to hard-wired values used for + specific instrument configurations. + + Args: + scifile (:obj:`str`): + File to use when determining the configuration and how + to adjust the input parameters. + inp_par (:class:`~pypeit.par.parset.ParSet`, optional): + Parameter set used for the full run of PypeIt. If None, + use :func:`default_pypeit_par`. + + Returns: + :class:`~pypeit.par.parset.ParSet`: The PypeIt parameter set + adjusted for configuration specific parameter values. + """ + par = super().config_specific_par(scifile, inp_par=inp_par) + + if par['calibrations']['wavelengths']['reid_arxiv'] is None: + msgs.warn("Wavelength setup not supported!" + msgs.newline() + msgs.newline() + + "Please perform your own wavelength calibration, and provide the path+filename using:" + msgs.newline() + + msgs.pypeitpar_text(['calibrations', 'wavelengths', 'reid_arxiv = '])) + # Return + return par diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index fc47b5a89f..3c861056aa 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -108,6 +108,7 @@ class GeminiGMOSSpectrograph(spectrograph.Spectrograph): """ ndet = 3 url = 'http://www.gemini.edu/instrumentation/gmos' + allowed_extensions = ['.fits', '.fits.bz2', '.fits.gz'] def __init__(self): super().__init__() @@ -1045,6 +1046,7 @@ class GeminiGMOSNSpectrograph(GeminiGMOSSpectrograph): telescope = telescopes.GeminiNTelescopePar() camera = 'GMOS-N' header_name = 'GMOS-N' + allowed_extensions = ['.fits', '.fits.bz2', '.fits.gz'] class GeminiGMOSNHamSpectrograph(GeminiGMOSNSpectrograph): diff --git a/pypeit/spectrographs/gemini_gnirs.py b/pypeit/spectrographs/gemini_gnirs.py index 5a88298847..52e346661b 100644 --- a/pypeit/spectrographs/gemini_gnirs.py +++ b/pypeit/spectrographs/gemini_gnirs.py @@ -24,6 +24,7 @@ class GeminiGNIRSSpectrograph(spectrograph.Spectrograph): url = 'https://www.gemini.edu/instrumentation/gnirs' header_name = 'GNIRS' telescope = telescopes.GeminiNTelescopePar() + allowed_extensions = ['.fits', '.fits.bz2'] def __init__(self): super().__init__() diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index cdf4a20b10..abb0bd1179 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -860,7 +860,7 @@ def get_detector_par(self, det, hdu=None): # Some properties of the image binning = self.compound_meta(self.get_headarr(hdu), "binning") numamps = hdu[0].header['NVIDINP'] - specflip = True if hdu[0].header['AMPID1'] == 2 else False + specflip = False if hdu[0].header['AMPMODE'] == 'ALL' else True gainmul, gainarr = hdu[0].header['GAINMUL'], np.zeros(numamps) ronarr = np.zeros(numamps) # Set this to zero (determine the readout noise from the overscan regions) # dsecarr = np.array(['']*numamps) diff --git a/pypeit/spectrographs/mdm_modspec.py b/pypeit/spectrographs/mdm_modspec.py index 68b1dbe2c6..25f0f78077 100644 --- a/pypeit/spectrographs/mdm_modspec.py +++ b/pypeit/spectrographs/mdm_modspec.py @@ -31,6 +31,8 @@ class MDMModspecEchelleSpectrograph(spectrograph.Spectrograph): supported = True comment = 'MDM Modspec spectrometer; Only 1200l/mm disperser (so far)' + allowed_extensions = ['.fit'] + def get_detector_par(self, det, hdu=None): """ Return metadata for the selected detector. diff --git a/pypeit/spectrographs/spectrograph.py b/pypeit/spectrographs/spectrograph.py index 4cfca1fc98..46779d8f8d 100644 --- a/pypeit/spectrographs/spectrograph.py +++ b/pypeit/spectrographs/spectrograph.py @@ -147,7 +147,7 @@ class Spectrograph: Metadata model that is generic to all spectrographs. """ - allowed_extensions = None + allowed_extensions = ['.fits', '.fits.gz'] """ Defines the allowed extensions for the input fits files. """ @@ -271,6 +271,44 @@ def ql_par(): ) ) ) + + @classmethod + def find_raw_files(cls, root, extension=None): + """ + Find raw observations for this spectrograph in the provided directory. + + This is a wrapper for :func:`~pypeit.io.files_from_extension` that + handles the restrictions of the file extensions specific to this + spectrograph. + + Args: + root (:obj:`str`, `Path`_, :obj:`list`): + One or more paths to search for files, which may or may not include + the prefix of the files to search for. For string input, this can + be the directory ``'/path/to/files/'`` or the directory plus the + file prefix ``'/path/to/files/prefix'``, which yeilds the search + strings ``'/path/to/files/*fits'`` or + ``'/path/to/files/prefix*fits'``, respectively. For a list input, + this can use wildcards for multiple directories. + extension (:obj:`str`, :obj:`list`, optional): + One or more file extensions to search on. If None, uses + :attr:`allowed_extensions`. Otherwise, this *must* be a subset + of the allowed extensions for the selected spectrograph. + + Returns: + :obj:`list`: List of `Path`_ objects with the full path to the set of + unique raw data filenames that match the provided criteria search + strings. + """ + if extension is None: + _ext = cls.allowed_extensions + else: + _ext = [extension] if isinstance(extension, str) else extension + _ext = [e for e in _ext if e in cls.allowed_extensions] + if len(_ext) == 0: + msgs.error(f'{extension} is not or does not include allowed extensions for ' + f'{cls.name}; choose from {cls.allowed_extensions}.') + return io.files_from_extension(root, extension=_ext) def _check_extensions(self, filename): """ @@ -282,9 +320,9 @@ def _check_extensions(self, filename): """ if self.allowed_extensions is not None: _filename = Path(filename).absolute() - if _filename.suffix not in self.allowed_extensions: + if not any([_filename.name.endswith(ext) for ext in self.allowed_extensions]): msgs.error(f'The input file ({_filename.name}) does not have a recognized ' - f'extension ({_filename.suffix}). The allowed extensions for ' + f'extension. The allowed extensions for ' f'{self.name} include {",".join(self.allowed_extensions)}.') def _check_telescope(self): diff --git a/pypeit/spectrographs/wht_isis.py b/pypeit/spectrographs/wht_isis.py index e107264ac3..7464e459b8 100644 --- a/pypeit/spectrographs/wht_isis.py +++ b/pypeit/spectrographs/wht_isis.py @@ -125,6 +125,7 @@ class WHTISISBlueSpectrograph(WHTISISSpectrograph): name = 'wht_isis_blue' camera = 'ISISb' comment = 'Blue camera' + allowed_extensions = ['.fit', '.fit.gz'] def get_detector_par(self, det, hdu=None): """ diff --git a/pypeit/telescopes.py b/pypeit/telescopes.py index e8bc8c3d3a..32e9cbff80 100644 --- a/pypeit/telescopes.py +++ b/pypeit/telescopes.py @@ -10,6 +10,18 @@ #TODO: Remove 'Par' from class name? + +class AATTelescopePar(TelescopePar): + def __init__(self): + loc = EarthLocation.of_site('Siding Spring Observatory') + super(AATTelescopePar, self).__init__(name='AAT', + longitude=loc.lon.to(units.deg).value, + latitude=loc.lat.to(units.deg).value, + elevation=loc.height.to(units.m).value, + diameter=3.9, + eff_aperture=12.0) + + class GTCTelescopePar(TelescopePar): def __init__(self): loc = EarthLocation.of_site('Roque de los Muchachos') diff --git a/pypeit/tests/test_arc.py b/pypeit/tests/test_arc.py index fef470534e..3cde322d37 100644 --- a/pypeit/tests/test_arc.py +++ b/pypeit/tests/test_arc.py @@ -2,10 +2,12 @@ Module to run tests on ararclines """ import pytest +import numpy as np from pypeit.core import arc from pypeit import io + def test_detect_lines(): # Using Paranal night sky as an 'arc' arx_sky = io.load_sky_spectrum('paranal_sky.fits') diff --git a/pypeit/tests/test_calibrations.py b/pypeit/tests/test_calibrations.py index 3705674b59..52ac1958ac 100644 --- a/pypeit/tests/test_calibrations.py +++ b/pypeit/tests/test_calibrations.py @@ -1,13 +1,10 @@ -""" -Module to run tests on FlatField class -Requires files in Development suite and an Environmental variable -""" from pathlib import Path -import os import yaml import pytest import shutil +from IPython import embed + import numpy as np from pypeit import dataPaths @@ -16,9 +13,8 @@ from pypeit.images import buildimage from pypeit.par import pypeitpar from pypeit.spectrographs.util import load_spectrograph -from IPython import embed -from pypeit.tests.tstutils import dummy_fitstbl, data_output_path +from pypeit.tests.tstutils import data_output_path @pytest.fixture def fitstbl(): diff --git a/pypeit/tests/test_io.py b/pypeit/tests/test_io.py index 95d03abb14..2a6f226297 100644 --- a/pypeit/tests/test_io.py +++ b/pypeit/tests/test_io.py @@ -48,5 +48,5 @@ def test_grab_rawfiles(): _raw_files = inputfiles.grab_rawfiles(raw_paths=[str(root)], extension='.fits.gz') assert len(_raw_files) == 9, 'Found the wrong number of files' - assert all([str(root / f) in _raw_files for f in tbl['filename']]), 'Missing expected files' + assert all([root / f in _raw_files for f in tbl['filename']]), 'Missing expected files' diff --git a/pypeit/tests/test_metadata.py b/pypeit/tests/test_metadata.py index 048191cc3f..467daa44bf 100644 --- a/pypeit/tests/test_metadata.py +++ b/pypeit/tests/test_metadata.py @@ -30,7 +30,7 @@ def test_read_combid(): # Generate the pypeit file with the comb_id droot = tstutils.data_output_path('b') pargs = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blue', '-c', 'all', '-b', - '--extension', 'fits.gz', '--output_path', f'{config_dir.parent}']) + '--output_path', f'{config_dir.parent}']) Setup.main(pargs) pypeit_file = config_dir / 'shane_kast_blue_A.pypeit' diff --git a/pypeit/tests/test_setups.py b/pypeit/tests/test_setups.py index e4356968e1..5759072024 100644 --- a/pypeit/tests/test_setups.py +++ b/pypeit/tests/test_setups.py @@ -63,12 +63,12 @@ def test_run_setup(): droot = tstutils.data_output_path('b') odir = Path(tstutils.data_output_path('')).absolute() / 'shane_kast_blue_A' pargs = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blue', '-c', 'all', - '--extension', 'fits.gz', '--output_path', f'{odir.parent}']) + '--output_path', f'{odir.parent}']) Setup.main(pargs) # Fails because name of spectrograph is wrong pargs2 = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blu', '-c', 'all', - '--extension', 'fits.gz', '--output_path', f'{odir.parent}']) + '--output_path', f'{odir.parent}']) with pytest.raises(ValueError): Setup.main(pargs2)