From 72e15eeb21d367a3216bbde68d4b1c2dd3ade7ab Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 20 Sep 2023 11:10:34 -0500 Subject: [PATCH 1/4] Overhaul build process with setuptools_scm and written version file --- .gitignore | 2 ++ .pre-commit-config.yaml | 2 +- MANIFEST.in | 15 +++++++--- doc/conf.py | 65 +++------------------------------------- doc/rtd_environment.yaml | 6 ++++ pyproject.toml | 7 +++++ pyspectral/__init__.py | 12 ++++---- setup.cfg | 6 ---- setup.py | 14 ++------- 9 files changed, 39 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index 51acbc76..e0321e4e 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ nosetests.xml .mr.developer.cfg .project .pydevproject + +pyspectral/version.py \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b170e1d6..46363c2b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - id: flake8 additional_dependencies: [flake8-docstrings, flake8-debugger, flake8-bugbear] - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort language_version: python3 diff --git a/MANIFEST.in b/MANIFEST.in index 67974bbe..4e87c71b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,13 @@ -include pyspectral/etc/* -include pyspectral/data/e490_00a.dat -include pyspectral/data/MSG_SEVIRI_Spectral_Response_Characterisation.XLS +prune * +exclude * +graft doc +recursive-exclude doc/_build * +graft pyspectral +recursive-exclude pyspectral/data/modis * include LICENSE.txt -include versioneer.py +include README.md +include AUTHORS.md +include CHANGELOG.md +include CHANGELOG_RSR_DATA.md include pyspectral/version.py +global-exclude *.py[cod] diff --git a/doc/conf.py b/doc/conf.py index f9a744e6..47ecf74e 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,19 +1,8 @@ # -*- coding: utf-8 -*- -# -# Pyspectral documentation build configuration file, created by -# sphinx-quickstart on Tue Oct 15 13:31:45 2013. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. +"""Sphinx documentation configuration file.""" -import sys import os -from pkg_resources import get_distribution +import sys # PYTHONPATH = docs/source DOC_SOURCES_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -24,62 +13,16 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('../')) -sys.path.insert(0, os.path.abspath('../pyspectral')) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -release = get_distribution('pyspectral').version +from pyspectral import __version__ as release # noqa + # for example take major/minor version = '.'.join(release.split('.')[:2]) - -class Mock(object): - - """Mock class for doc/unit testing (on Travis)""" - - __all__ = [] - - def __init__(self, *args, **kwargs): - pass - - def __call__(self, *args, **kwargs): - return Mock() - - @classmethod - def __getattr__(cls, name): - if name in ('__file__', '__path__'): - return '/dev/null' - elif name[0] == name[0].upper(): - mockType = type(name, (), {}) - mockType.__module__ = __name__ - return mockType - elif name == "inf": - return 0 - else: - return Mock() - - -MOCK_MODULES = ['numpy', 'numpy.core', - 'numpy.distutils.core', 'numpy.core.multiarray', - 'dask', - 'scipy', 'scipy.integrate', 'scipy.interpolate', - 'scipy.interpolate.InterpolatedUnivariateSpline', - 'geotiepoints', 'geotiepoints.multilinear', - 'trollsift', 'trollsift.parser', - 'h5py', 'tqdm', 'xlrd'] - -for mod_name in MOCK_MODULES: - sys.modules[mod_name] = Mock() - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, '/opt/lib/python2.5/site-packages') -sys.path.insert(0, os.path.abspath('../pyspectral')) - - # -- General configuration ----------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. diff --git a/doc/rtd_environment.yaml b/doc/rtd_environment.yaml index d4b05526..5964381a 100644 --- a/doc/rtd_environment.yaml +++ b/doc/rtd_environment.yaml @@ -8,6 +8,12 @@ dependencies: - sphinx - sphinx_rtd_theme - sphinxcontrib-apidoc + - dask + - scipy + - trollsift + - h5py + - tqdm + - xlrd - pip: - graphviz - .. # relative path to the satpy project diff --git a/pyproject.toml b/pyproject.toml index d5941825..6d2ebe94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,10 @@ +[build-system] +requires = ["setuptools>=60", "wheel", "setuptools_scm[toml]>=8.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "pyspectral/version.py" + [tool.isort] sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"] profile = "black" diff --git a/pyspectral/__init__.py b/pyspectral/__init__.py index 66ae77f8..816de2f0 100644 --- a/pyspectral/__init__.py +++ b/pyspectral/__init__.py @@ -19,10 +19,10 @@ """Pyspectral package init.""" -from pkg_resources import DistributionNotFound, get_distribution - try: - __version__ = get_distribution(__name__).version -except DistributionNotFound: - # package is not installed - pass + from pyspectral.version import version as __version__ # noqa +except ModuleNotFoundError: + raise ModuleNotFoundError( + "No module named pyspectral.version. This could mean " + "you didn't install 'pyspectral' properly. Try reinstalling ('pip " + "install').") diff --git a/setup.cfg b/setup.cfg index 12800174..4959a976 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,3 @@ -[options] -setup_requires = - setuptools_scm - setuptools_scm_git_archive - - [bdist_rpm] provides=pyspectral requires=numpy h5py python-geotiepoints dask PyYAML python-requests python-appdirs diff --git a/setup.py b/setup.py index 292301e6..51bebfff 100644 --- a/setup.py +++ b/setup.py @@ -23,20 +23,11 @@ from setuptools import find_packages, setup -try: - # HACK: https://github.com/pypa/setuptools_scm/issues/190#issuecomment-351181286 - # Stop setuptools_scm from including all repository files - import setuptools_scm.integration - setuptools_scm.integration.find_files = lambda _: [] -except ImportError: - pass - - description = ('Reading and manipulaing satellite sensor spectral responses and the ' 'solar spectrum, to perfom various corrections to VIS and NIR band data') try: - with open('./README', 'r') as fd: + with open('./README.md', 'r') as fd: long_description = fd.read() except IOError: long_description = '' @@ -62,8 +53,8 @@ 'Topic :: Scientific/Engineering'], url='https://github.com/pytroll/pyspectral', long_description=long_description, + long_description_content_type="text/markdown", license='GPLv3', - packages=find_packages(), include_package_data=True, package_data={ @@ -93,5 +84,4 @@ tests_require=test_requires, python_requires='>=3.7', zip_safe=False, - use_scm_version=True ) From 0f0b6aa7dd0e2a534e714d9fb806e1fe4f50b215 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 20 Sep 2023 11:26:06 -0500 Subject: [PATCH 2/4] Fix flake8 issues --- plot_some_band.py | 7 ++++--- pyspectral/tests/test_rayleigh.py | 1 - pyspectral/utils.py | 3 ++- rsr_convert_scripts/agri_rsr.py | 1 - rsr_convert_scripts/msu_gs_reader.py | 20 +++++++++++--------- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/plot_some_band.py b/plot_some_band.py index 8f854cd7..a4cf9cf2 100644 --- a/plot_some_band.py +++ b/plot_some_band.py @@ -19,14 +19,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - -"""Plot relative spectral responses for a list of sensors""" +"""Plot relative spectral responses for a list of sensors.""" import argparse + import matplotlib.pyplot as plt +import numpy as np + from pyspectral.rsr_reader import RelativeSpectralResponse from pyspectral.utils import get_bandname_from_wavelength -import numpy as np platforms = ['Himawari-8', 'GOES-16', 'Meteosat-10', 'EOS-Aqua', 'Sentinel-3A', 'Sentinel-3A', diff --git a/pyspectral/tests/test_rayleigh.py b/pyspectral/tests/test_rayleigh.py index 0059bf44..40765942 100644 --- a/pyspectral/tests/test_rayleigh.py +++ b/pyspectral/tests/test_rayleigh.py @@ -282,7 +282,6 @@ def test_rayleigh_reduction(self, fake_lut_hdf5): def test_rayleigh_getname(self): """Test logic for Rayleigh instrument selection.""" - with pytest.raises(ValueError): _create_rayleigh(platform='FY-4B') diff --git a/pyspectral/utils.py b/pyspectral/utils.py index da3afe04..2945ab13 100644 --- a/pyspectral/utils.py +++ b/pyspectral/utils.py @@ -383,7 +383,8 @@ def download_luts(aerosol_types=None, dry_run=False, aerosol_type=None): def _get_aerosol_types(aerosol_types, aerosol_type): if aerosol_type is not None: - warnings.warn("'aerosol_type' is deprecated, use 'aerosol_types' instead.", UserWarning) + warnings.warn("'aerosol_type' is deprecated, use 'aerosol_types' instead.", UserWarning, + stacklevel=3) if isinstance(aerosol_type, (list, tuple, set)): aerosol_types = aerosol_type else: diff --git a/rsr_convert_scripts/agri_rsr.py b/rsr_convert_scripts/agri_rsr.py index 5bf4a2dd..5958e4f4 100644 --- a/rsr_convert_scripts/agri_rsr.py +++ b/rsr_convert_scripts/agri_rsr.py @@ -80,7 +80,6 @@ def __init__(self, bandname, platform_name): self._get_options_from_config() self._get_bandfilenames() - LOG.debug("Filenames: %s", str(self.filenames)) if self.filenames[bandname] and os.path.exists(self.filenames[bandname]): self.requested_band_filename = self.filenames[bandname] diff --git a/rsr_convert_scripts/msu_gs_reader.py b/rsr_convert_scripts/msu_gs_reader.py index ef71ccb2..56217fed 100644 --- a/rsr_convert_scripts/msu_gs_reader.py +++ b/rsr_convert_scripts/msu_gs_reader.py @@ -15,17 +15,20 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +"""Read the Electro-L N2 MSU-GS spectral response functions. -"""Read the Electro-L N2 MSU-GS spectral response functions. Data from the NWPSAF: +Data from the NWPSAF: https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/ir_srf/rtcoef_electro-l_2_msugs_srf.html """ +import logging import os + import numpy as np -from pyspectral.utils import convert2hdf5 as tohdf5 + from pyspectral.raw_reader import InstrumentRSR -import logging +from pyspectral.utils import convert2hdf5 as tohdf5 LOG = logging.getLogger(__name__) @@ -40,12 +43,11 @@ class MsugsRSR(InstrumentRSR): - """Container for the Electro-L N2 MSU-GS relative spectral response data""" + """Container for the Electro-L N2 MSU-GS relative spectral response data.""" def __init__(self, bandname, platform_name): - - super(MsugsRSR, self).__init__( - bandname, platform_name, MSUGS_BAND_NAMES) + """Load files.""" + super(MsugsRSR, self).__init__(bandname, platform_name, MSUGS_BAND_NAMES) self.instrument = 'msu-gs' self._get_options_from_config() @@ -67,7 +69,7 @@ def __init__(self, bandname, platform_name): self.wavespace = 'wavelength' def _load(self, scale=10000.0): - """Load the MSU-GS RSR data for the band requested""" + """Load the MSU-GS RSR data for the band requested.""" data = np.genfromtxt(self.requested_band_filename, unpack=True, names=['wavenumber', @@ -84,7 +86,7 @@ def _load(self, scale=10000.0): def main(): - """Main""" + """Run main script.""" for platform_name in ['Electro-L-N2', ]: tohdf5(MsugsRSR, platform_name, MSUGS_BAND_NAMES) From 863ab130b9456b890a600a5f94d78ff0c7218a7d Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 20 Sep 2023 12:54:07 -0500 Subject: [PATCH 3/4] Update pyspectral/__init__.py Co-authored-by: Martin Raspaud --- pyspectral/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyspectral/__init__.py b/pyspectral/__init__.py index 816de2f0..9d6e29d7 100644 --- a/pyspectral/__init__.py +++ b/pyspectral/__init__.py @@ -25,4 +25,4 @@ raise ModuleNotFoundError( "No module named pyspectral.version. This could mean " "you didn't install 'pyspectral' properly. Try reinstalling ('pip " - "install').") + "install pyspectral').") From 9a21305fa661208c9ad3ec5b89684f263f30f111 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 17:57:20 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyspectral/tests/test_utils.py | 2 +- rsr_convert_scripts/avhrr1_rsr.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pyspectral/tests/test_utils.py b/pyspectral/tests/test_utils.py index b0ee8927..3969bc00 100644 --- a/pyspectral/tests/test_utils.py +++ b/pyspectral/tests/test_utils.py @@ -33,7 +33,7 @@ import responses from pyspectral import utils -from pyspectral.utils import bytes2string, check_and_adjust_instrument_name, are_instruments_identical, np2str +from pyspectral.utils import are_instruments_identical, bytes2string, check_and_adjust_instrument_name, np2str TEST_RSR = {'20': {}, } TEST_RSR2 = {'20': {}, } diff --git a/rsr_convert_scripts/avhrr1_rsr.py b/rsr_convert_scripts/avhrr1_rsr.py index 5fd45c93..f6de9ad6 100644 --- a/rsr_convert_scripts/avhrr1_rsr.py +++ b/rsr_convert_scripts/avhrr1_rsr.py @@ -22,15 +22,16 @@ Data from NOAA: AVHRR1_SRF_only.xls """ +import logging import os -from xlrd import open_workbook -from pyspectral.config import get_config -from pyspectral.utils import get_central_wave + +import h5py import numpy as np import pkg_resources -import logging -import h5py +from xlrd import open_workbook +from pyspectral.config import get_config +from pyspectral.utils import get_central_wave LOG = logging.getLogger(__name__)