diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b69812608..4a375c71c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -26,24 +26,24 @@ jobs: - name: Tests, Python 3.12, Windows os: windows-latest - noxenv: tests-3.12 + noxenv: tests python: '3.12' noxposargs: --durations=10 - name: Tests, Python 3.11, Windows os: windows-latest - noxenv: tests-3.11 + noxenv: tests python: '3.11' noxposargs: --durations=10 - name: Tests, Python 3.10, macOS os: macos-latest - noxenv: tests-3.10 + noxenv: tests python: '3.10' - name: Tests, Python 3.10, Linux os: ubuntu-latest - noxenv: tests-3.10 + noxenv: tests python: '3.10' - name: Import XRTpy, Python 3.10, Linux @@ -101,4 +101,4 @@ jobs: run: sudo apt install graphviz - name: Build documentation - run: nox -s build_docs_nitpicky -- -q + run: nox -s docs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20e425cab..1b1399ad4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-ast name: validate Python code @@ -23,7 +23,7 @@ repos: - id: check-yaml - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.4 hooks: - id: check-github-workflows @@ -47,11 +47,6 @@ repos: args: [--autofix] - id: pretty-format-yaml args: [--autofix] - # For the labeler GitHub Action, labels with spaces in them must - # be put in quotes. However, the pretty-format-yaml hook will - # remove the quotes which will break that action (and certain other - # actions). - exclude: .github/labeler.yml|.pre-commit-search-and-replace.yaml - repo: https://github.com/MarcoGorelli/absolufy-imports rev: v0.3.1 @@ -62,8 +57,6 @@ repos: - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: - - id: python-check-blanket-noqa - name: noqa comments have an error code - id: rst-directive-colons - id: rst-inline-touching-normal - id: text-unicode-replacement-char @@ -79,30 +72,27 @@ repos: exclude: .*\.fits - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.3 + rev: v0.7.0 hooks: - id: ruff - name: ruff (see https://docs.astral.sh/ruff/rules) + name: ruff args: [--fix] - id: ruff-format name: autoformat source code with ruff formatter - repo: https://github.com/asottile/blacken-docs - rev: 1.18.0 + rev: 1.19.0 hooks: - id: blacken-docs name: autoformat code blocks in docs additional_dependencies: - - black==24.1.1 + - black - repo: https://github.com/nbQA-dev/nbQA rev: 1.8.7 hooks: - id: nbqa-check-ast name: validate Python notebooks - - id: nbqa-ruff - name: ruff for notebooks (see https://docs.astral.sh/ruff/rules) - args: [--fix, '--select=A,ARG,B,BLE,C,C4,E,F,FLY,I,INT,ISC,PERF,PIE,PLC,PLE,PYI,Q003,RET,RSE,SIM,TID,TRY,UP,W', '--ignore=B018,E402,E501,PLC2401,TRY003'] - id: nbqa-black additional_dependencies: - - black==24.1.1 + - black diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e9ce913a6..c79487fac 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,9 +7,9 @@ formats: - htmlzip build: - os: ubuntu-22.04 + os: ubuntu-lts-latest tools: - python: '3.11' + python: latest apt_packages: - graphviz jobs: diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 000000000..9830ef2b7 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,79 @@ +target-version = "py310" +show-fixes = true +extend-exclude = [ + ".jupyter", + "__pycache__", + "_build", + "_dev", +] + +[lint] +# Find info about ruff rules at: https://docs.astral.sh/ruff/rules +extend-select = [ + "ARG", # flake8-unused-arguments + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "C90", # mccabe + "COM818", # trailing-comma-on-bare-tuple + "FBT003", # flake8-boolean-trap + "FLY", # flynt + "I", # isort + "ICN", # flake8-import-conventions + "INP", # flake8-no-pep420 + "INT", # flake8-gettext + "ISC", # flake8-implicit-str-concat + "N", # pep8-naming + "NPY", # numpy-deprecated-type-alias + "PD", # pandas-vet + "PERF", # perflint + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PLC", # pylint convention + "PLE", # pylint errors + "PLW", # pylint warnings + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "RSE", # flake8-raise + "RUF005",# collection-literal-concatenation + "RUF006", # asyncio-dangling-task + "RUF007", # pairwise-over-zipped + "RUF008", # mutable-dataclass-default + "RUF009", # function-call-in-dataclass-default-argument + "RUF010", # explicit-f-string-type-conversion + "RUF013", # implicit-optional + "RUF015", # unnecessary-iterable-allocation-for-first-element + "RUF016", # invalid-index-type + "RUF100", # unused-noqa + "RUF200", # invalid-pyproject-toml + "S", # flake8-bandit + "SIM", # flake8-simplify + "TCH", # flake8-type-checking + "TID", # flake8-tidy-imports + "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle warnings +] +ignore = [ + "C901", # is too complex + "E501", # line-too-long + "ISC001", # single-line-implicit-string-concatenation (formatter conflict) + "N802", # invalid-function-name + "N803", # invalid-argument-name + "N806", # non-lowercase-variable-in-function + "N816", # mixed-case-variable-in-global-scope + "PLC2401", # non-ascii-name + "S101", # asserts + "SIM108", # if-else-block-instead-of-if-exp + "TRY003", # raise-vanilla-args +] + +[lint.per-file-ignores] +"docs/conf.py" = [ + "E402", # Module imports not at top of file + "INP001", # Implicit-namespace-package. The examples are not a package. +] +"examples/*" = [ + "INP001", # Implicit-namespace-package. The examples are not a package. +] diff --git a/README.md b/README.md index ad41ed28d..22d002d78 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ [![Read the Docs Status](https://readthedocs.org/projects/xrtpy/badge/?version=latest&logo=twitter)](http://xrtpy.readthedocs.io/en/latest/?badge=latest) [![astropy](http://img.shields.io/badge/powered%20by-AstroPy-orange.svg?style=flat&logo=astropy)](http://www.astropy.org/) -XRTpy is a Python package being developed for the analysis of observations made by the X-Ray Telescope (XRT) on the *Hinode* spacecraft. +XRTpy is a Python package being developed for the analysis of observations made by the X-Ray Telescope (XRT) on the **Hinode** spacecraft. ## Acknowledgements -The development of XRTpy is supported by NASA contract NNM07AB07C to the Smithsonian Astrophysical Observatory. +The development of XRTpy is supported by NASA contract **NNM07AB07C** to the Smithsonian Astrophysical Observatory. diff --git a/docs/conf.py b/docs/conf.py index 786e15e3e..10c35df26 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,9 +21,9 @@ os.environ["PARFIVE_HIDE_PROGRESS"] = "True" # -- Imports ------------------------------------------------------------------- -from astropy.utils.exceptions import AstropyDeprecationWarning # NOQA: E402 -from matplotlib import MatplotlibDeprecationWarning # NOQA: E402 -from sunpy.util.exceptions import ( # NOQA: E402 +from astropy.utils.exceptions import AstropyDeprecationWarning +from matplotlib import MatplotlibDeprecationWarning +from sunpy.util.exceptions import ( SunpyDeprecationWarning, SunpyPendingDeprecationWarning, ) @@ -34,7 +34,7 @@ copyright = f"2021-{datetime.now(tz=UTC).year}, {author}" # The full version, including alpha/beta/rc tags -from xrtpy import __version__ # NOQA: E402 +from xrtpy import __version__ _version_ = Version(__version__) # NOTE: Avoid "post" appearing in version string in rendered docs @@ -85,11 +85,9 @@ # ones. extensions = [ "hoverxref.extension", - "matplotlib.sphinxext.plot_directive", "sphinx_automodapi.automodapi", "sphinx_automodapi.smart_resolver", "sphinx_copybutton", - "sphinx_design", "sphinx_gallery.gen_gallery", "sphinx_issues", "sphinx.ext.autodoc", @@ -176,7 +174,6 @@ # -- Options for intersphinx extension --------------------------------------- -# Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { "python": ( "https://docs.python.org/3/", diff --git a/examples/effective_area.py b/examples/effective_area.py index 467f7c940..f2052cfd4 100644 --- a/examples/effective_area.py +++ b/examples/effective_area.py @@ -53,12 +53,12 @@ plt.figure() plt.plot( - eaf.channel_wavelength, + eaf.wavelength, effective_area, label=f"{date_time}", ) plt.plot( - eaf.channel_wavelength, + eaf.wavelength, launch_effective_area, label=f"{relative_launch_date_time}", ) diff --git a/noxfile.py b/noxfile.py index 5944dd8dc..3a01c7833 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,69 +1,53 @@ import nox -nox.options.sessions = ["tests", "linters"] - +nox.options.sessions = ["tests"] python_versions = ("3.10", "3.11", "3.12") -sphinx_paths = ["docs", "docs/_build/html"] -sphinx_fail_on_warnings = ["-W", "--keep-going"] -sphinx_builder = ["-b", "html"] -sphinx_opts = sphinx_paths + sphinx_fail_on_warnings + sphinx_builder -sphinx_no_notebooks = ["-D", "nbsphinx_execute=never"] -sphinx_nitpicky = ["-n"] - -pytest_options = [ - "--ignore", - "xrtpy/response/effective_area.py", - "--ignore", - "xrtpy/response/temperature_response.py", -] - -@nox.session(python=python_versions) +@nox.session def tests(session): - session.install(".[dev,tests]") + """ + Run tests with pytest. + """ + pytest_options = {} + session.install(".[tests]") session.run("pytest", *pytest_options) @nox.session def linters(session): + """ + Run all pre-commit hooks on all files. + """ session.install("pre-commit") session.run("pre-commit", "run", "--all-files", *session.posargs) @nox.session def import_package(session): + """ + Import xrtpy. + """ session.install(".") - session.run("python", "-c", 'import xrtpy') # fmt: skip + session.run("python", "-c", "import xrtpy") @nox.session -def build_docs(session): - session.install(".[dev,docs]") - session.run( - "sphinx-build", - *sphinx_opts, - *session.posargs, +def docs(session): + """ + Build documentation with Sphinx. + """ + sphinx_paths = ["docs", "docs/_build/html"] + sphinx_fail_on_warnings = ["-W", "--keep-going"] + sphinx_builder = ["-b", "html"] + sphinx_nitpicky = ["-n"] + sphinx_opts = ( + sphinx_paths + sphinx_fail_on_warnings + sphinx_builder + sphinx_nitpicky ) - - -@nox.session -def build_docs_nitpicky(session): - session.install(".[dev,docs]") + session.install(".[docs]") session.run( "sphinx-build", *sphinx_opts, *sphinx_nitpicky, *session.posargs, ) - - -@nox.session -def build_docs_no_examples(session): - session.install(".[dev,docs]") - session.run( - "sphinx-build", - *sphinx_opts, - *sphinx_no_notebooks, - *session.posargs, - ) diff --git a/pyproject.toml b/pyproject.toml index 711546d00..b15a4d96f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,15 @@ [build-system] build-backend = "setuptools.build_meta" requires = [ - "setuptools>=50", - "setuptools_scm>=6", - "wheel>=0.34", + "setuptools >=62.1 , != 71.0.1", + "setuptools_scm[toml] >=6.2", + "wheel >=0.34", ] [project] name = "XRTpy" readme = "README.md" -keywords = ["solar physics"] +keywords = ["Solar Physics", "x-ray", "Hinode", "XRT"] description = "For analyzing data from the X-Ray Telescope (XRT) on the Hinode spacecraft." license = {file = "LICENSE"} classifiers = [ @@ -21,7 +21,6 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering :: Astronomy", "Topic :: Scientific/Engineering :: Physics", ] @@ -34,44 +33,46 @@ authors = [ {name = "Jonathan Slavin", email = "jslavin@cfa.harvard.edu"}, {name = "Nick Murpy", email="namurphy@cfa.harvard.edu"}, {name = "Will Barnes"}, + {name = "Nabil Freij", email="nabil.freij@gmail.com"}, {name = "Stuart Mumford"}, ] dependencies = [ - "astropy >= 5.1", - "cached-property >= 1.5.2", + "astropy >= 5.3.0", "matplotlib >= 3.5.0", - "numpy >= 1.24.0", - "requests >= 2.28.0", + "numpy >=1.23.5", "scikit-image >= 0.19.0", - "scipy >= 1.8.0", - # install setuptools to get pkg_resources - "setuptools; python_version >= '3.12'", - "sunpy[map] >= 4.0.0", + # !=1.10.0 due to https://github.com/scipy/scipy/issues/17718 + "scipy >= 1.9.0 , != 1.10.0", + "sunpy[map] >= 5.0.0", ] [project.optional-dependencies] dev = [ + "xrtpy[tests,docs]", "nox >= 2022.8.7", - "pre-commit >= 3.6.0", ] tests = [ "pytest >= 8.0.0", - "pytest-allclose >= 1.0.0", + "pytest-astropy", "pytest-xdist >= 3.6.1", ] docs = [ "pydata_sphinx_theme >= 0.15.0", + # Need pkg_resources for sphinxcontrib-bibtex + # and this comes from setuptools + # This is a bug upstream in the extension which + # has yet to be fixed. + "setuptools", "sphinx >= 7.3.0", "sphinx_automodapi >= 0.17.0", "sphinx-copybutton >= 0.5.2", - "sphinx-design>=0.2.0", "sphinx-gallery >= 0.16.0", "sphinx-hoverxref >= 1.4.0", "sphinx-issues >= 4.1.0", "sphinxcontrib-bibtex >= 2.6.2", "sphinxext-opengraph>=0.6.0", - "sunpy[net] >= 4.0.0", + "sunpy[net] >= 5.0.0", ] [project.urls] @@ -80,95 +81,22 @@ Repository = "https://github.com/HinodeXRT/xrtpy" Issues = "https://github.com/HinodeXRT/xrtpy/issues" Changelog = "https://xrtpy.readthedocs.io/en/stable/changelog/index.html" -[tool.ruff] -target-version = "py310" -show-fixes = true -extend-exclude = [ - ".jupyter", - "__pycache__", - "_build", - "_dev", -] -namespace-packages = [".github/workflows", "docs"] - -[tool.ruff.lint] -# Find info about ruff rules at: https://docs.astral.sh/ruff/rules -extend-select = [ - "ARG", # flake8-unused-arguments - "B", # flake8-bugbear - "BLE", # flake8-blind-except - "C4", # flake8-comprehensions - "C90", # mccabe - "COM818", # trailing-comma-on-bare-tuple - "FBT003", # flake8-boolean-trap - "FLY", # flynt - "I", # isort - "ICN", # flake8-import-conventions - "INP", # flake8-no-pep420 - "INT", # flake8-gettext - "ISC", # flake8-implicit-str-concat - "N", # pep8-naming - "NPY", # numpy-deprecated-type-alias - "PD", # pandas-vet - "PERF", # perflint - "PGH", # pygrep-hooks - "PIE", # flake8-pie - "PLC", # pylint convention - "PLE", # pylint errors - "PLW", # pylint warnings - "PT", # flake8-pytest-style - "PTH", # flake8-use-pathlib - "PYI", # flake8-pyi - "RSE", # flake8-raise - "RUF005",# collection-literal-concatenation - "RUF006", # asyncio-dangling-task - "RUF007", # pairwise-over-zipped - "RUF008", # mutable-dataclass-default - "RUF009", # function-call-in-dataclass-default-argument - "RUF010", # explicit-f-string-type-conversion - "RUF013", # implicit-optional - "RUF015", # unnecessary-iterable-allocation-for-first-element - "RUF016", # invalid-index-type - "RUF100", # unused-noqa - "RUF200", # invalid-pyproject-toml - "S", # flake8-bandit - "SIM", # flake8-simplify - "TCH", # flake8-type-checking - "TID", # flake8-tidy-imports - "TRY", # tryceratops - "UP", # pyupgrade - "W", # pycodestyle warnings -] -ignore = [ - "E501", # line-too-long - "ISC001", # single-line-implicit-string-concatenation (formatter conflict) - "N802", # invalid-function-name - "N803", # invalid-argument-name - "N806", # non-lowercase-variable-in-function - "N816", # mixed-case-variable-in-global-scope - "PLC2401", # non-ascii-name - "S101", # asserts - "SIM108", # if-else-block-instead-of-if-exp - "TRY003", # raise-vanilla-args -] - -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["E402", "F401", "F402", "F403"] # ignore import errors -"examples/*" = ["INP001"] # is part of an implicit namespace package. +[tool.setuptools] +packages = ["xrtpy"] -[tool.ruff.lint.flake8-import-conventions.aliases] -"astropy.units" = "u" -"matplotlib.pyplot" = "plt" -numpy = "np" -pandas = "pd" +[tool.setuptools.package-data] +"xrtpy" = ["data/*"] +"xrtpy.response" = ["data/*.txt", "data/*.geny"] +"xrtpy.response.tests" = ["data/*/*/*.txt"] -[tool.ruff.lint.mccabe] -max-complexity = 12 +[tool.setuptools_scm] +write_to = "xrtpy/version.py" [tool.codespell] skip = "*.genx,*.geny,*.png,*egg*,.git,.hypothesis,.nox,.tox,.idea,__pycache__,_build" ignore-words-list = """ 4rd, +aas, bu, circularly, egde, @@ -176,83 +104,8 @@ fo, nd, ons, sav, +sav, te, tne, -ue, -aas +ue """ - -[tool.pytest.ini_options] -testpaths = ['xrtpy', 'docs'] -xfail_strict = true -doctest_optionflags = """ -NORMALIZE_WHITESPACE -ELLIPSIS -NUMBER -IGNORE_EXCEPTION_DETAIL""" -norecursedirs = [ - 'build', - 'docs/_build', - 'examples', - 'auto_examples', -] -addopts = [ - '--doctest-modules', - '--doctest-continue-on-failure', - '--ignore=docs/conf.py', -] - -[tool.setuptools] -packages = ["xrtpy"] - -[tool.setuptools.package-data] -"xrtpy" = ["data/*"] -"xrtpy.response" = ["data/*.txt", "data/*.geny"] -"xrtpy.response.tests" = ["data/*/*/*.txt"] - -[tool.setuptools_scm] -write_to = "xrtpy/version.py" - -[tool.towncrier] -package = "xrtpy" -name = "xrtpy" -filename = "CHANGELOG.rst" -directory = "changelog/" -title_format = "{name} v{version} ({project_date})" -issue_format = ":pr:`{issue}`" # Despite the name mismatch, we use this for linking to PRs -wrap = true - -[[tool.towncrier.type]] -directory = "breaking" -name = "Backwards Incompatible Changes" -showcontent = true - -[[tool.towncrier.type]] -directory = "removal" -name = "Deprecations and Removals" -showcontent = true - -[[tool.towncrier.type]] -directory = "feature" -name = "Features" -showcontent = true - -[[tool.towncrier.type]] -directory = "bugfix" -name = "Bug Fixes" -showcontent = true - -[[tool.towncrier.type]] -directory = "doc" -name = "Improved Documentation" -showcontent = true - -[[tool.towncrier.type]] -directory = "trivial" -name = "Trivial/Internal Changes" -showcontent = true - -[tool.gilesbot] - -[tool.gilesbot.pull_requests] -enabled = true diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..984a006cd --- /dev/null +++ b/pytest.ini @@ -0,0 +1,24 @@ +[pytest] +minversion = 8.0 +testpaths = + xrtpy + docs +xfail_strict = true +norecursedirs = + build + docs/_build + examples + auto_examples +doctest_plus = enabled +doctest_optionflags = + NORMALIZE_WHITESPACE + FLOAT_CMP + ELLIPSIS + IGNORE_EXCEPTION_DETAIL +addopts = + --doctest-rst + -p no:unraisableexception + -p no:theadexception + --arraydiff + --doctest-ignore-import-errors + --doctest-continue-on-failure diff --git a/xrtpy/__init__.py b/xrtpy/__init__.py index 6ffc3a2f6..ba3646bec 100644 --- a/xrtpy/__init__.py +++ b/xrtpy/__init__.py @@ -10,8 +10,8 @@ try: from xrtpy.version import __version__ except ImportError: - warnings.warn("version not found.") # noqa: B028 - + warnings.warn("version not found.", stacklevel=3) + __version__ = "0.0.0" # Then you can be explicit to control what ends up in the namespace, -__all__ = ["response"] +__all__ = ["response", "__version__"] diff --git a/xrtpy/image_correction/remove_lightleak.py b/xrtpy/image_correction/remove_lightleak.py index 40473a414..38fe9450d 100644 --- a/xrtpy/image_correction/remove_lightleak.py +++ b/xrtpy/image_correction/remove_lightleak.py @@ -86,9 +86,10 @@ def _get_stray_light_phase(date_obs): phase = 0 if phase == 6: - warnings.warn( # noqa: B028 + warnings.warn( "light leak images for this period are not yet" - " available. Defaulting to previous phase." + " available. Defaulting to previous phase.", + stacklevel=3, ) phase = 5 @@ -203,7 +204,10 @@ def remove_lightleak(in_map, scale=1.0, leak_map=None): images at the phase 1 (as of Feb-2022). """ if "Light leak subtraction: DONE" in in_map.meta["HISTORY"]: - warnings.warn("HISTORY indicates light leak subtraction already done on image.") # noqa: B028 + warnings.warn( + "HISTORY indicates light leak subtraction already done on image.", + stacklevel=3, + ) if leak_map is None: fw1 = in_map.meta["EC_FW1_"] diff --git a/xrtpy/image_correction/tests/test_remove_lightleak.py b/xrtpy/image_correction/tests/test_remove_lightleak.py index d01855139..7a1f8c92d 100644 --- a/xrtpy/image_correction/tests/test_remove_lightleak.py +++ b/xrtpy/image_correction/tests/test_remove_lightleak.py @@ -1,61 +1,34 @@ from pathlib import Path +import numpy as np import pytest +from astropy.utils.data import get_pkg_data_path from sunpy.map import Map from xrtpy.image_correction.remove_lightleak import remove_lightleak - -def get_IDL_data_file(): - """ - The XRT composite fits file that been lightleak corrected in - IDL have been rename to begin with "ll" referring to lightleak. - """ - directory = ( - Path(__file__).parent.absolute() - / "data" - / "light_leak_testing_data_files" - / "IDL_lightleak_corrected_data_test_files" +data_dir = Path( + get_pkg_data_path( + "data/light_leak_testing_data_files", package="xrtpy.image_correction.tests" ) - data_files = directory.glob("ll_comp_XRT*.fits") - return sorted(data_files) - - -IDL_filenames = get_IDL_data_file() - - -def get_composite_data_files(): - """ - The XRT composite fits file are no corrected in IDL. - These files will be corrected using XRTpy. - """ - directory = ( - Path(__file__).parent.absolute() - / "data" - / "light_leak_testing_data_files" - / "xrtpy_lightleak_data_test_files" - ) - data_file = directory.glob("comp_XRT*.fits") - return sorted(data_file) - - -composite_filenames = get_composite_data_files() - -# Using zip as an iterator to pair the data files together. Trouble-free method to use in pytest-parametrize -data_files = list(zip(IDL_filenames, composite_filenames, strict=False)) - - -@pytest.mark.parametrize(("idlfile", "compfile"), data_files) -def test_lightleak(idlfile, compfile, allclose): +) +composite_filenames = sorted( + (data_dir / "xrtpy_lightleak_data_test_files").glob("comp_XRT*.fits") +) +IDL_filenames = sorted( + (data_dir / "IDL_lightleak_corrected_data_test_files").glob("ll_comp_XRT*.fits") +) + + +@pytest.mark.parametrize( + ("idlfile", "compfile"), list(zip(IDL_filenames, composite_filenames, strict=True)) +) +def test_lightleak(idlfile, compfile): IDL_map = Map(idlfile) input_map = Map(compfile) - ll_removed_map_xrtpy = remove_lightleak(input_map) - - if input_map.data.shape == (2048, 2048): - # Because of rebinning for full resolution images, the match is worse - # between IDL created images and XRTpy ones. IDL's method of rebinning - # is different from that used by sunpy. - assert allclose(ll_removed_map_xrtpy.data, IDL_map.data, atol=0.75) - else: - assert allclose(ll_removed_map_xrtpy.data, IDL_map.data, atol=1e-5) + # Because of rebinning for full resolution images, the match is worse + # between IDL created images and XRTpy ones. IDL's method of rebinning + # is different from that used by sunpy. + atol = 0.75 if input_map.data.shape == (2048, 2048) else 1e-5 + assert np.allclose(ll_removed_map_xrtpy.data, IDL_map.data, atol=atol) diff --git a/xrtpy/response/channel.py b/xrtpy/response/channel.py index 6a11cb984..dbbace117 100644 --- a/xrtpy/response/channel.py +++ b/xrtpy/response/channel.py @@ -430,6 +430,8 @@ def ccd_gain_left(self) -> u.electron / u.DN: @u.quantity_input def ccd_gain_right(self) -> u.electron / u.DN: """Gain when reading the right port of the CCD.""" + # NOTE: Value for the right gain in the instrument data files is incorrect. + # See https://github.com/HinodeXRT/xrtpy/pull/76 return u.Quantity(57.5, u.electron / u.DN) @property diff --git a/xrtpy/response/data/xrt_contam_on_ccd.geny b/xrtpy/response/data/xrt_contam_on_ccd.geny index 045e15aa8..3e4e44c24 100644 Binary files a/xrtpy/response/data/xrt_contam_on_ccd.geny and b/xrtpy/response/data/xrt_contam_on_ccd.geny differ diff --git a/xrtpy/response/effective_area.py b/xrtpy/response/effective_area.py index 468bafd98..dd7a43bcb 100644 --- a/xrtpy/response/effective_area.py +++ b/xrtpy/response/effective_area.py @@ -4,12 +4,12 @@ import datetime import math -import os from functools import cached_property from pathlib import Path import astropy.time import numpy as np +import scipy.interpolate import scipy.io import sunpy.io.special import sunpy.time @@ -114,7 +114,7 @@ def observation_date(self, date): f"Date must be after {epoch}." ) - modified_time_path = os.path.getmtime(_ccd_contam_filename) # noqa: PTH204 + modified_time_path = Path(_ccd_contam_filename).stat().st_mtime modified_time = astropy.time.Time(modified_time_path, format="unix") latest_available_ccd_data = _ccd_contamination_file_time[-1].datetime.strftime( "%Y/%m/%d" @@ -343,7 +343,7 @@ def n_DEHP_attributes(self): "data/n_DEHP.txt", package="xrtpy.response" ) - with open(_n_DEHP_filename) as n_DEHP: # noqa: PTH123 + with Path(_n_DEHP_filename).open() as n_DEHP: list_of_DEHP_attributes = [] for line in n_DEHP: stripped_line = line.strip() @@ -522,29 +522,34 @@ def _CCD_contamination_transmission(self): return np.array([abs(transmittance[i] ** 2) for i in range(4000)]) @property - def channel_wavelength(self): + def wavelength(self): """Array of wavelengths for every X-ray channel in Angstroms (Å).""" - return Channel(self.name).wavelength + _wave = self._channel.wavelength.to_value("AA") + delta_wave = 0.01 + return np.arange(_wave[0], _wave[-1], delta_wave) * u.Angstrom @property def channel_geometry_aperture_area(self): """XRT flight model geometry aperture area.""" - return Channel(self.name).geometry.geometry_aperture_area + return self._channel.geometry.geometry_aperture_area @property def channel_transmission(self): """XRT channel transmission.""" - return Channel(self.name).transmission + return np.interp( + self.wavelength, self._channel.wavelength, self._channel.transmission + ) + + def _contamination_interpolator(self, x, y): + return np.interp(self.wavelength.to_value("Angstrom"), x, y) @property def _interpolated_CCD_contamination_transmission(self): """Interpolate filter contam transmission to the wavelength.""" - CCD_contam_transmission = np.interp( - self.channel_wavelength.to_value("AA"), + return self._contamination_interpolator( self.n_DEHP_wavelength, self._CCD_contamination_transmission, ) - return CCD_contam_transmission @cached_property def _filter_contamination_transmission(self): @@ -586,12 +591,10 @@ def _filter_contamination_transmission(self): @property def _interpolated_filter_contamination_transmission(self): """Interpolate filter contam transmission to the wavelength.""" - Filter_contam_transmission = np.interp( - self.channel_wavelength.to_value("AA"), + return self._contamination_interpolator( self.n_DEHP_wavelength, self._filter_contamination_transmission, ) - return Filter_contam_transmission @u.quantity_input def effective_area(self) -> u.cm**2: diff --git a/xrtpy/response/temperature_from_filter_ratio.py b/xrtpy/response/temperature_from_filter_ratio.py index 8ec7f45f6..c6cfab4dc 100644 --- a/xrtpy/response/temperature_from_filter_ratio.py +++ b/xrtpy/response/temperature_from_filter_ratio.py @@ -4,8 +4,8 @@ """ import logging -from collections import namedtuple from datetime import datetime +from typing import Any, NamedTuple import numpy as np from astropy import units as u @@ -18,10 +18,15 @@ __all__ = ["temperature_from_filter_ratio"] -TempEMdata = namedtuple("TempEMdata", "Tmap, EMmap, Terrmap, EMerrmap") # noqa: PYI024 +class TempEMdata(NamedTuple): + Tmap: Any + EMmap: Any + Terrmap: Any + EMerrmap: Any -def temperature_from_filter_ratio( # noqa: C901 + +def temperature_from_filter_ratio( map1, map2, abundance_model="coronal", @@ -287,7 +292,7 @@ def temperature_from_filter_ratio( # noqa: C901 logging.info(f"Examined T_e range: {Tmodel.min():.3E} - {Tmodel.max():.3E} K") logging.info("No thresholds applied") Tmap, EMmap, Terrmap, EMerrmap = make_results_maps( - hdr1, hdr2, T_e, EM, T_error, EMerror, mask + hdr1, hdr2, T_e, EM, T_error, EMerror ) return TempEMdata(Tmap, EMmap, Terrmap, EMerrmap) @@ -530,7 +535,7 @@ def calculate_TE_errors(map1, map2, T_e, EM, model_ratio, tresp1, tresp2, Trange Narukage's K factor for image 2 """ - wvl = tresp1.channel_wavelength + wvl = tresp1.wavelength eVe = tresp1.ev_per_electron gain = tresp1.ccd_gain_right # (h*c/lambda) * 1/(eV per electron) * 1/gain @@ -605,7 +610,7 @@ def calculate_TE_errors(map1, map2, T_e, EM, model_ratio, tresp1, tresp2, Trange return T_error, EMerror, K1, K2 -def make_results_maps(hdr1, hdr2, T_e, EM, T_error, EMerror, mask): # noqa: ARG001 +def make_results_maps(hdr1, hdr2, T_e, EM, T_error, EMerror): """ Create SunPy Map objects from the image metadata and temperature, volume emission measure, temperature uncertainty and emission measure uncertainty @@ -632,10 +637,6 @@ def make_results_maps(hdr1, hdr2, T_e, EM, T_error, EMerror, mask): # noqa: ARG EMerror : 2D float array image containing the uncertainties in EM derived for the images - mask : 2D boolean array - image containing the mask for T_e and EM, either provided or derived - from the data - Returns: -------- Tmap : ~sunpy.map.sources.hinode.XRTMap diff --git a/xrtpy/response/temperature_response.py b/xrtpy/response/temperature_response.py index ed880bdf3..b32fbfbf6 100644 --- a/xrtpy/response/temperature_response.py +++ b/xrtpy/response/temperature_response.py @@ -4,19 +4,15 @@ from pathlib import Path +import astropy.constants as const import numpy as np import scipy.io from astropy import units as u -from astropy.constants import c, h from scipy import interpolate from xrtpy.response.channel import Channel, resolve_filter_name from xrtpy.response.effective_area import EffectiveAreaFundamental -_c_Å_per_s = c.to(u.angstrom / u.second).value -_h_eV_s = h.to(u.eV * u.s).value - - _abundance_model_file_path = { "coronal_abundance_path": Path(__file__).parent.absolute() / "data/chianti_emission_models" @@ -158,31 +154,31 @@ def file_spectra(self): @property @u.quantity_input - def wavelength(self): + def _wavelength_spectra(self): """Emission model file wavelength values in Å.""" return u.Quantity(self._get_abundance_data["wavelength"] * u.Angstrom) @property @u.quantity_input - def channel_wavelength(self): + def wavelength(self): """Array of wavelengths for every X-ray channel in Å.""" - return u.Quantity((Channel(self.filter_name).wavelength[:3993]) * u.photon) + return self._effective_area_fundamental.wavelength @property def focal_len(self): """Focal length of the telescope in units of cm.""" - return Channel(self.filter_name).geometry.geometry_focal_len + return self._channel.geometry.geometry_focal_len @property def ev_per_electron(self): """Amount of energy it takes to dislodge 1 electron in the CCD.""" - return Channel(self.filter_name).ccd.ccd_energy_per_electron + return self._channel.ccd.ccd_energy_per_electron @property @u.quantity_input def pixel_size(self) -> u.cm: """CCD pixel size. Units converted from μm to cm.""" - ccd_pixel_size = Channel(self.filter_name).ccd.ccd_pixel_size + ccd_pixel_size = self._channel.ccd.ccd_pixel_size return ccd_pixel_size.to(u.cm) @property @@ -204,14 +200,12 @@ def spectra(self) -> u.photon * u.cm**3 / (u.sr * u.s * u.Angstrom): spectra_interpolate = [] for i in range(61): interpolater = interpolate.interp1d( - self.wavelength, + self._wavelength_spectra.to_value("AA"), self.file_spectra[i], kind="linear", ) - spectra_interpolate.append(interpolater(self.channel_wavelength)) - return spectra_interpolate * ( - u.photon * u.cm**3 * (1 / u.sr) * (1 / u.s) * (1 / u.Angstrom) - ) + spectra_interpolate.append(interpolater(self.wavelength.to_value("AA"))) + return spectra_interpolate * u.Unit("photon cm3 sr-1 s-1 Angstrom-1") @u.quantity_input def effective_area(self) -> u.cm**2: @@ -235,27 +229,24 @@ def integration(self) -> u.electron * u.cm**5 / (u.s * u.pix): astropy.units.Quantity Integrated temperature response in electron cm^5 / (s pix). """ - wavelength = (self.channel_wavelength).value - constants = (_c_Å_per_s * _h_eV_s / self.channel_wavelength).value - factors = (self.solid_angle_per_pixel / self.ev_per_electron).value - effective_area = (self.effective_area()).value - dwvl = wavelength[1:] - wavelength[:-1] - dwvl = np.append(dwvl, dwvl[-1]) + constants = const.h * const.c / self.wavelength / u.photon + constants *= self.solid_angle_per_pixel / self.ev_per_electron # Simple summing like this is appropriate for binned data like in the current # spectrum file. More recent versions of Chianti include the line width, # which then makes the previous version that uses Simpson's method # to integrate more appropriate (10/05/2022) - temp_resp_w_u_c = ( - self.spectra().value * effective_area * constants * factors * dwvl + return ( + self.spectra() + * self.effective_area() + * constants + * np.gradient(self.wavelength) ).sum(axis=1) - return temp_resp_w_u_c * (u.electron * u.cm**5 * (1 / u.s) * (1 / u.pix)) - @property @u.quantity_input def ccd_gain_right(self) -> u.electron / u.DN: """Provide the camera gain in electrons per data number.""" - return Channel(self.filter_name).ccd.ccd_gain_right + return self._channel.ccd.ccd_gain_right @u.quantity_input def temperature_response(self) -> u.DN * u.cm**5 / (u.s * u.pix): diff --git a/xrtpy/response/tests/test_channel.py b/xrtpy/response/tests/test_channel.py index d9172f816..e7edb16ed 100644 --- a/xrtpy/response/tests/test_channel.py +++ b/xrtpy/response/tests/test_channel.py @@ -848,41 +848,6 @@ def test_channel_name2(channel_name): assert name == IDL_mirror_name_AUTO -@pytest.mark.parametrize("channel_name", channel_names) -def test_channel_wavelength(channel_name): - channel_filter = Channel(channel_name) - - wavelength_length = int(channel_filter.number_of_wavelengths) - wavelength = channel_filter.wavelength[:wavelength_length] - - idl_array_length = int( - v6_genx_s[_channel_name_to_index_mapping[channel_name]]["LENGTH"] - ) - idl_wavelength_auto = ( - v6_genx_s[_channel_name_to_index_mapping[channel_name]]["WAVE"][ - :idl_array_length - ] - * u.angstrom - ) - - assert u.allclose(idl_wavelength_auto, wavelength) - - idl_mirror_wavelength_manu = [ - 9.00000, - 9.10000, - 9.20000, - 9.30000, - 9.40000, - 9.50000, - 9.60000, - 9.70000, - 9.80000, - 9.90000, - ] * u.angstrom - - assert u.allclose(idl_mirror_wavelength_manu, wavelength[80:90]) - - @pytest.mark.parametrize("channel_name", channel_names) def test_channel_transmission(channel_name): channel_filter = Channel(channel_name) diff --git a/xrtpy/response/tests/test_effective_area.py b/xrtpy/response/tests/test_effective_area.py index 750eeed95..f1c6d23e7 100644 --- a/xrtpy/response/tests/test_effective_area.py +++ b/xrtpy/response/tests/test_effective_area.py @@ -1,8 +1,10 @@ from datetime import datetime from pathlib import Path +import numpy as np import pytest from astropy import units as u +from astropy.utils.data import get_pkg_data_filenames from xrtpy.response.channel import Channel from xrtpy.response.effective_area import EffectiveAreaFundamental @@ -70,9 +72,7 @@ def test_EffectiveArea_filter_name(name): instance = EffectiveAreaFundamental( name, datetime(year=2013, month=9, day=22, hour=22, minute=0, second=0) ) - actual_attr_value = instance.name - - assert actual_attr_value == name + assert instance.name == name @pytest.mark.parametrize("date", valid_dates) @@ -92,81 +92,41 @@ def test_EffectiveArea_contamination_on_filter(name, date): @pytest.mark.parametrize("date", invalid_dates) @pytest.mark.parametrize("name", channel_names) def test_EffectiveArea_exception_is_raised(name, date): - with pytest.raises(ValueError): # noqa: PT011 + with pytest.raises(ValueError, match="Invalid date"): EffectiveAreaFundamental(name, date) def get_IDL_data_files(): - directory = ( - Path(__file__).parent.parent.absolute() - / "data" - / "effective_area_IDL_testing_files" - ) - filter_data_files = directory.glob("**/*.txt") + filter_data_files = [] + for dir in get_pkg_data_filenames( + "data/effective_area_IDL_testing_files", package="xrtpy.response.tests" + ): + filter_data_files += list(Path(dir).glob("*.txt")) return sorted(filter_data_files) -filenames = get_IDL_data_files() - - -def _IDL_raw_data_list(filename): - with open(filename) as filter_file: # noqa: PTH123 - list_of_IDL_effective_area_data = [] - for line in filter_file: - stripped_line = line.strip() - line_list = stripped_line.split() - list_of_IDL_effective_area_data.append(line_list) - - return list_of_IDL_effective_area_data - - -def IDL_test_filter_name(list_of_lists): - return str(list_of_lists[0][1]) - - -def IDL_test_date(list_of_lists): - obs_date = str(list_of_lists[1][1]) - obs_time = str(list_of_lists[1][2]) - - day = int(obs_date[:2]) - - month_datetime_object = datetime.strptime(obs_date[3:6], "%b") - month = month_datetime_object.month - - year = int(obs_date[8:12]) - - hour = int(obs_time[:2]) - minute = int(obs_time[3:5]) - second = int(obs_time[6:8]) - - return datetime(year, month, day, hour, minute, second) - - -def _IDL_effective_area_raw_data(filename): - with open(filename) as filter_file: # noqa: PTH123 - list_of_lists = [] - for line in filter_file: - stripped_line = line.strip() - line_list = stripped_line.split() - list_of_lists.append(line_list) - - effective_area = [list_of_lists[i][1] for i in range(3, len(list_of_lists))] - effective_area = [float(i) for i in effective_area] * u.cm**2 - - return effective_area - - -@pytest.mark.parametrize("filename", filenames) -def test_EffectiveAreaPreparatory_effective_area(filename, allclose): - data_list = _IDL_raw_data_list(filename) - - filter_name = IDL_test_filter_name(data_list) - filter_obs_date = IDL_test_date(data_list) - - IDL_effective_area = _IDL_effective_area_raw_data(filename) - +# NOTE: This is marked as xfail because the IDL results that this test compares against +# are incorrect due to the use of quadratic interpolation in the contamination curves +# which leads to ringing near the edges in the contamination curve. +# See https://github.com/HinodeXRT/xrtpy/pull/284#issuecomment-2334503108 +@pytest.mark.xfail +@pytest.mark.parametrize("filename", get_IDL_data_files()) +def test_effective_area_compare_idl(filename): + with Path.open(filename) as f: + filter_name = f.readline().split()[1] + filter_obs_date = " ".join(f.readline().split()[1:]) + # NOTE: Annoyingly the date strings use "Sept" instead of "Sep" for "September" + filter_obs_date = filter_obs_date.replace("Sept", "Sep") + IDL_data = np.loadtxt(filename, skiprows=3) + IDL_wavelength = IDL_data[:, 0] * u.AA + IDL_effective_area = IDL_data[:, 1] * u.cm**2 instance = EffectiveAreaFundamental(filter_name, filter_obs_date) actual_effective_area = instance.effective_area() - - assert actual_effective_area.unit == IDL_effective_area.unit - assert allclose(actual_effective_area.value, IDL_effective_area.value, atol=1e-2) + IDL_effective_area = np.interp( + instance.wavelength, IDL_wavelength, IDL_effective_area + ) + assert u.allclose( + actual_effective_area, + IDL_effective_area, + rtol=1e-6, + ) diff --git a/xrtpy/response/tests/test_temperature_response.py b/xrtpy/response/tests/test_temperature_response.py index c9fedfaee..e98bc47e7 100644 --- a/xrtpy/response/tests/test_temperature_response.py +++ b/xrtpy/response/tests/test_temperature_response.py @@ -1,86 +1,67 @@ -from datetime import datetime from pathlib import Path +import astropy.units as u +import numpy as np import pytest +from astropy.utils.data import get_pkg_data_filenames from xrtpy.response.temperature_response import TemperatureResponseFundamental -def get_IDL_data_files(model): - path = ( - Path(__file__).parent.parent.absolute() - / "tests" - / "data" - / f"temperature_response_{model}_IDL_testing_files" - ) - filter_data_files = list(path.glob("**/*.txt")) +def get_IDL_data_files(abundance): + filter_data_files = [] + for dir in get_pkg_data_filenames( + f"data/temperature_response_{abundance}_IDL_testing_files", + package="xrtpy.response.tests", + ): + filter_data_files += list(Path(dir).glob("*.txt")) return sorted(filter_data_files) -def _IDL_raw_data_list(filename): - with open(filename) as filter_file: # noqa: PTH123 - IDL_data_list = [] - for line in filter_file: - stripped_line = line.strip() - line_list = stripped_line.split() - IDL_data_list.append(line_list) - return IDL_data_list - - -def IDL_test_abundance_name(IDL_data_list): - return str(IDL_data_list[1][1]) - - -def IDL_test_filter_name(IDL_data_list): - return str(IDL_data_list[2][1]) - - -def IDL_test_date(IDL_data_list): - obs_date = str(IDL_data_list[3][1]) - obs_time = str(IDL_data_list[3][2]) - - day = int(obs_date[:2]) - month_datetime_object = datetime.strptime(obs_date[3:6], "%b") - month = month_datetime_object.month - year = int(obs_date[8:12]) - - hour = int(obs_time[:2]) - minute = int(obs_time[3:5]) - second = int(obs_time[6:8]) - return datetime(year, month, day, hour, minute, second) - - -def _IDL_temperature_response_raw_data(filename): - with open(filename) as filter_file: # noqa: PTH123 - IDL_data_list = [] - for line in filter_file: - stripped_line = line.strip() - line_list = stripped_line.split() - IDL_data_list.append(line_list) - - new_IDL_data_list = [IDL_data_list[i][1] for i in range(5, len(IDL_data_list))] - return [float(i) for i in new_IDL_data_list] - +filenames = ( + get_IDL_data_files("coronal") + + get_IDL_data_files("hybrid") + + get_IDL_data_files("photospheric") +) + + +@pytest.mark.parametrize("filename", filenames) +def test_temperature_response(filename): + with Path.open(filename) as f: + _ = f.readline() + abundance = f.readline().split()[1] + filter_name = f.readline().split()[1] + filter_obs_date = " ".join(f.readline().split()[1:]) + # NOTE: Annoyingly the date strings use "Sept" instead of "Sep" for "September" + filter_obs_date = filter_obs_date.replace("Sept", "Sep") + IDL_data = np.loadtxt(filename, skiprows=5) + IDL_temperature = IDL_data[:, 0] * u.K + IDL_temperature_response = IDL_data[:, 1] * u.Unit("DN cm5 pix-1 s-1") + + instance = TemperatureResponseFundamental( + filter_name, filter_obs_date, abundance_model=abundance + ) -@pytest.mark.parametrize("abundance_model", ["coronal", "hybrid", "photospheric"]) -def test_temperature_response(abundance_model, allclose): - filenames = get_IDL_data_files(abundance_model) - for filename in filenames: - IDL_data = _IDL_raw_data_list(filename) - filter_name = IDL_test_filter_name(IDL_data) - filter_obs_date = IDL_test_date(IDL_data) - IDL_temperature_response = _IDL_temperature_response_raw_data(filename) + IDL_temperature_response = np.interp( + instance.CHIANTI_temperature, IDL_temperature, IDL_temperature_response + ) - instance = TemperatureResponseFundamental( - filter_name, filter_obs_date, abundance_model=abundance_model - ) + actual_temperature_response = instance.temperature_response() - actual_temperature_response = instance.temperature_response() - atol = actual_temperature_response.value.max() * 0.013 + # NOTE: there may be small deviations where the response function is very small, likely + # due to differences in the interpolation schemes. These are not critical as the response + # is effectively zero in these regions anyway. + i_valid = np.where( + actual_temperature_response > 1e-8 * actual_temperature_response.max() + ) - assert allclose( - actual_temperature_response.value, - IDL_temperature_response, - rtol=0.028, - atol=atol, - ) + # NOTE: The relative tolerance is set comparatively high here because the CCD right gain + # values in the IDL and xrtpy codes are explicitly different. See https://github.com/HinodeXRT/xrtpy/pull/76. + # If the IDL results files are corrected to account for this updated gain, then this + # relative tolerance can be set significantly lower. Setting the gains to be the same, + # nearly all cases match within less than 1%. + assert u.allclose( + actual_temperature_response[i_valid], + IDL_temperature_response[i_valid], + rtol=5e-2, + ) diff --git a/xrtpy/util/filename2repo_path.py b/xrtpy/util/filename2repo_path.py index 5cf513b38..94bb88596 100644 --- a/xrtpy/util/filename2repo_path.py +++ b/xrtpy/util/filename2repo_path.py @@ -2,7 +2,7 @@ from pathlib import Path -def filename2repo_path( # noqa: C901 +def filename2repo_path( filename, urlroot="https://xrt.cfa.harvard.edu/", join=False, verbose=False ): """