From aafcd64433eabebb72f2ba37da29be87662c4bf2 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Sun, 18 Dec 2022 09:25:59 -0500 Subject: [PATCH 1/5] try to expand test coverage expand test coverage fix test failure for dask trivial whitespace change? try to emulate https://github.com/astropy/astroquery/pull/2626 that last one seems to maybe have broken CI? undoing the CI changes try putting things on one line ? try changing ubuntu-latest to linux based on the versions-manifest, which doesn't mention ubuntu at all try changing 3.10 -> '3.10' instead of the linux change shorten job names so we can see the whole name in the sidebar of gh-actions add a mask setter to solve problem where numpy masked arrays are trying to set the mask attribute but being blocked from doing so drop openfiles b/c it's deprecated (not sure what we're supposed to replace it with yet) make some of the failing tests more extensive and explicit --- .github/workflows/main.yml | 28 +++++++++++++++++--------- setup.cfg | 4 ++-- spectral_cube/base_class.py | 4 ++++ spectral_cube/tests/test_dask.py | 2 ++ spectral_cube/tests/test_projection.py | 9 +++++++++ tox.ini | 18 ++++++----------- 6 files changed, 41 insertions(+), 24 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42ba1b8ed..730160c94 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,43 +17,51 @@ jobs: include: - os: ubuntu-latest python-version: 3.9 - name: Python 3.9 with minimal dependencies + name: Py3.9 mindeps toxenv: py39-test + - os: ubuntu-latest + python-version: '3.10' + name: Py3.10 mindeps + toxenv: py310-test + - os: ubuntu-latest + python-version: 3.11 + name: Py3.11 mindeps + toxenv: py311-test - os: ubuntu-latest python-version: 3.9 - name: Python 3.9 with no visualization + coverage + name: Py3.9 no visualization + coverage toxenv: py39-test-novis-cov - os: ubuntu-latest python-version: 3.8 - name: Python 3.8 with minimal dependencies + name: Py3.8 mindeps toxenv: py38-test - os: ubuntu-latest python-version: 3.7 - name: Python 3.7 with minimal dependencies + name: Py3.7 mindeps toxenv: py37-test - os: ubuntu-latest python-version: 3.9 - name: Python 3.9 with all non-visualization dependencies (except CASA) + name: Py3.9 all noviz (except CASA) toxenv: py39-test-novis - os: ubuntu-18.04 python-version: 3.8 - name: Python 3.8 with minimal dependencies and CASA + name: Py3.8 mindeps and CASA toxenv: py38-test-casa - os: ubuntu-latest python-version: 3.9 - name: Python 3.9, all non-visualization dependencies, and dev versions of key dependencies + name: Py3.9, novis, dev toxenv: py39-test-novis-dev - os: ubuntu-latest python-version: 3.8 - name: Python 3.8 with all non-visualization dependencies (except CASA) + name: Py3.8 all novis (except CASA) toxenv: py38-test-novis - os: macos-latest python-version: 3.9 - name: Python 3.9 with all non-visualization dependencies (except CASA) on MacOS X + name: Py3.9 all noviz (except CASA) MacOS X toxenv: py39-test-novis - os: windows-latest python-version: 3.9 - name: Python 3.9, all dependencies, and dev versions of key dependencies on Windows + name: Py3.9, alldeps, dev; Windows toxenv: py39-test-all-dev-noopenfiles - os: ubuntu-latest python-version: 3.9 diff --git a/setup.cfg b/setup.cfg index 138342317..e738ef674 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,8 +54,8 @@ all = [options.package_data] spectral_cube.tests = - data/* - data/*/* + data/* + data/*/* spectral_cube.io.tests = data/*/* diff --git a/spectral_cube/base_class.py b/spectral_cube/base_class.py index 8faf07c9c..c02187ec5 100644 --- a/spectral_cube/base_class.py +++ b/spectral_cube/base_class.py @@ -52,6 +52,10 @@ def meta(self): def mask(self): return self._mask + @mask.setter + def mask(self, value): + self._mask = value + class HeaderMixinClass(object): """ A mixin class to provide header updating from WCS objects. diff --git a/spectral_cube/tests/test_dask.py b/spectral_cube/tests/test_dask.py index 272fe37b1..77316ce56 100644 --- a/spectral_cube/tests/test_dask.py +++ b/spectral_cube/tests/test_dask.py @@ -13,6 +13,8 @@ try: from distributed.utils_test import client, loop, cluster_fixture, cleanup, loop_in_thread # noqa + # zarr & fsspec required for writing to disk w/dask + import zarr, fsspec # noqa DISTRIBUTED_INSTALLED = True except ImportError: DISTRIBUTED_INSTALLED = False diff --git a/spectral_cube/tests/test_projection.py b/spectral_cube/tests/test_projection.py index 8bc7b93aa..e3419ae97 100644 --- a/spectral_cube/tests/test_projection.py +++ b/spectral_cube/tests/test_projection.py @@ -138,6 +138,9 @@ def test_self_arith(LDO, data): image = data p = LDO(image, copy=False) + p.wcs = WCS() + assert hasattr(p, '_wcs') + assert p.wcs is not None p2 = p + p @@ -161,12 +164,17 @@ def test_self_arith_with_beam(LDO, data): image = data p = LDO(image, copy=False) p = p.with_beam(exp_beam) + p.wcs = WCS() + assert hasattr(p, 'beam') + assert hasattr(p, '_wcs') + assert p.wcs is not None p2 = p + p assert hasattr(p2, '_wcs') assert p2.wcs == p.wcs assert np.all(p2.value==2) + assert hasattr(p2, 'beam') assert p2.beam == exp_beam p2 = p - p @@ -174,6 +182,7 @@ def test_self_arith_with_beam(LDO, data): assert hasattr(p2, '_wcs') assert p2.wcs == p.wcs assert np.all(p2.value==0) + assert hasattr(p2, 'beam') assert p2.beam == exp_beam diff --git a/tox.ini b/tox.ini index 8d412bad8..159c081f8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{37,38,39}-test{,-all,-dev,-novis,-cov,-noopenfiles} + py{37,38,39,310,311}-test{,-all,-dev,-novis,-cov,-noopenfiles} build_docs codestyle requires = @@ -10,15 +10,9 @@ set_env = casa: PIP_EXTRA_INDEX_URL = {env:PIP_EXTRA_INDEX_URL:https://casa-pip.nrao.edu/repository/pypi-group/simple} [testenv] -passenv = - HOME - DISPLAY - LC_ALL - LC_CTYPE -changedir = - .tmp/{envname} -description = - run tests with pytest +passenv = HOME,DISPLAY,LC_ALL,LC_CTYPE,ON_TRAVIS +changedir = .tmp/{envname} +description = run tests with pytest deps = dev: git+https://github.com/radio-astro-tools/pvextractor#egg=pvextractor dev: git+https://github.com/radio-astro-tools/radio-beam#egg=radio-beam @@ -35,9 +29,9 @@ extras = all: all commands = pip freeze - !cov-!noopenfiles: pytest --open-files --pyargs spectral_cube {toxinidir}/docs {posargs} + !cov-!noopenfiles: pytest --pyargs spectral_cube {toxinidir}/docs {posargs} noopenfiles: pytest --pyargs spectral_cube {toxinidir}/docs {posargs} - cov: pytest --open-files --pyargs spectral_cube {toxinidir}/docs --cov spectral_cube --cov-config={toxinidir}/setup.cfg {posargs} + cov: pytest --pyargs spectral_cube {toxinidir}/docs --cov spectral_cube --cov-config={toxinidir}/setup.cfg {posargs} cov: coverage xml -o {toxinidir}/coverage.xml [testenv:build_docs] From 1abb8216aba3766361c80fac8eb5e758dd517aa7 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Thu, 9 Mar 2023 21:08:30 -0500 Subject: [PATCH 2/5] split a test onto more lines to try to pin down where the error is coming from (pdb revealed that code that succeeded on the previous line is the direct source of this failure) fix mistake add a new test for "all" add all-dev and remove -openfiles from windoze move dev dependenc installs from tox to setup.cfg whitespace and naming expand test coverage yet again: try dev w/o novis/all separate out 'viz' from 'noviz' fix a typo.... add extras that were all missing into tox.ini make sure basic tests use latest numpy rearrange and make tests more granular still drop 3.7 from grid b/c astropy 5.2.1 doesn't support it see if pvextractor can be included in the "good" viz hack around astropy 14514 hack needs to be hackier b/c _new_view didn't always have finalize= option try to run tests w/dev branch fix syntax fix install url fix name again? try to fix last exception allow for three types of error? this might not be the right approach fix test again remove mask setter - I'm not sure it was actually needed, I think that might have been a side-effect error caused by something else? Pushing this to run tests to check; may neeed to partially revert this if tests fail set ndim fix the naxis defn restore the mask setter, it _is_ needed reproject high_level_wcs fix merged; revert to master in tests fix names of # of dims --- .github/workflows/main.yml | 48 +++++++++++++------ setup.cfg | 29 +++++------ spectral_cube/base_class.py | 1 + spectral_cube/lower_dimensional_structures.py | 11 ++++- spectral_cube/tests/test_projection.py | 13 ++--- spectral_cube/tests/test_spectral_cube.py | 5 +- tox.ini | 14 +++--- 7 files changed, 76 insertions(+), 45 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 730160c94..187d95b4c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,39 +30,59 @@ jobs: - os: ubuntu-latest python-version: 3.9 name: Py3.9 no visualization + coverage - toxenv: py39-test-novis-cov + toxenv: py39-test-noviz-cov - os: ubuntu-latest python-version: 3.8 name: Py3.8 mindeps toxenv: py38-test - os: ubuntu-latest - python-version: 3.7 - name: Py3.7 mindeps - toxenv: py37-test + python-version: 3.9 + name: Py3.9 noviz + toxenv: py39-test-noviz + - os: ubuntu-latest + python-version: 3.9 + name: Py3.9 all + toxenv: py39-test-viz-noviz-docs + - os: ubuntu-latest + python-version: 3.9 + name: Py3.9 dev + toxenv: py39-test-dev - os: ubuntu-latest python-version: 3.9 - name: Py3.9 all noviz (except CASA) - toxenv: py39-test-novis + name: Py3.9 viz + toxenv: py39-test-viz + #- os: ubuntu-latest + # python-version: 3.9 + # name: Py3.9 viz w/yt & glue + # toxenv: py39-test-viz-viz_extra + - os: ubuntu-latest + python-version: 3.9 + name: Py3.9 all dev + toxenv: py39-test-viz-noviz-docs-dev - os: ubuntu-18.04 python-version: 3.8 name: Py3.8 mindeps and CASA toxenv: py38-test-casa - os: ubuntu-latest python-version: 3.9 - name: Py3.9, novis, dev - toxenv: py39-test-novis-dev + name: Py3.9, noviz, dev + toxenv: py39-test-noviz-dev - os: ubuntu-latest python-version: 3.8 - name: Py3.8 all novis (except CASA) - toxenv: py38-test-novis + name: Py3.8 noviz (except CASA) + toxenv: py38-test-noviz - os: macos-latest python-version: 3.9 - name: Py3.9 all noviz (except CASA) MacOS X - toxenv: py39-test-novis + name: Py3.9 noviz (except CASA) MacOS X + toxenv: py39-test-noviz + - os: windows-latest + python-version: 3.9 + name: Py3.9, noviz Windows + toxenv: py39-test-noviz-dev - os: windows-latest python-version: 3.9 - name: Py3.9, alldeps, dev; Windows - toxenv: py39-test-all-dev-noopenfiles + name: Py3.9, noviz, dev; Windows + toxenv: py39-test-noviz-dev - os: ubuntu-latest python-version: 3.9 name: Documentation diff --git a/setup.cfg b/setup.cfg index e738ef674..34f944c94 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,32 +25,33 @@ install_requires = test = pytest-astropy pytest-cov + regions>=0.7 + numpy>=1.24.0 + astropy>=5.2.1 docs = sphinx-astropy matplotlib -novis = +noviz = zarr fsspec distributed pvextractor - regions>=0.3 - reproject + reproject>=0.9.1 scipy -all = - pytest-astropy - pytest-cov - sphinx-astropy - zarr - fsspec - distributed +viz = aplpy - glue-core[qt] matplotlib - pvextractor - regions>=0.3 reproject - scipy + pvextractor +viz_extra = + glue-core[qt] yt ; python_version<'3.8' +dev = + pvextractor @ git+https://github.com/radio-astro-tools/pvextractor#egg=pvextractor + radio-beam @ git+https://github.com/radio-astro-tools/radio-beam#egg=radio-beam + astropy @ git+https://github.com/astropy/astropy#egg=astropy + reproject @ git+https://github.com/astropy/reproject#egg=reproject + regions @ git+https://github.com/astropy/regions#egg=regions [options.package_data] spectral_cube.tests = diff --git a/spectral_cube/base_class.py b/spectral_cube/base_class.py index c02187ec5..566b22604 100644 --- a/spectral_cube/base_class.py +++ b/spectral_cube/base_class.py @@ -56,6 +56,7 @@ def mask(self): def mask(self, value): self._mask = value + class HeaderMixinClass(object): """ A mixin class to provide header updating from WCS objects. diff --git a/spectral_cube/lower_dimensional_structures.py b/spectral_cube/lower_dimensional_structures.py index 31975c189..031478752 100644 --- a/spectral_cube/lower_dimensional_structures.py +++ b/spectral_cube/lower_dimensional_structures.py @@ -33,6 +33,13 @@ class LowerDimensionalObject(u.Quantity, BaseNDClass, HeaderMixinClass): Generic class for 1D and 2D objects. """ + def _new_view(self, obj=None, unit=None, finalize=True): + # FORCE finalization to hack around https://github.com/astropy/astropy/issues/14514#issuecomment-1463935711 + try: + return super(LowerDimensionalObject, self)._new_view(obj=obj, unit=unit, finalize=True) + except TypeError: + return super(LowerDimensionalObject, self)._new_view(obj=obj, unit=unit) + @property def hdu(self): if self.wcs is None: @@ -246,7 +253,7 @@ def __new__(cls, value, unit=None, dtype=None, copy=True, wcs=None, raise ValueError("value should be a 2-d array") if wcs is not None and wcs.wcs.naxis != 2: - raise ValueError("wcs should have two dimension") + raise ValueError("wcs should have two dimensions") self = u.Quantity.__new__(cls, value, unit=unit, dtype=dtype, copy=copy).view(cls) @@ -599,7 +606,7 @@ def __new__(cls, value, unit=None, dtype=None, copy=True, wcs=None, raise ValueError("value should be a 1-d array") if wcs is not None and wcs.wcs.naxis != 1: - raise ValueError("wcs should have two dimension") + raise ValueError("wcs should have one dimension") self = u.Quantity.__new__(cls, value, unit=unit, dtype=dtype, copy=copy).view(cls) diff --git a/spectral_cube/tests/test_projection.py b/spectral_cube/tests/test_projection.py index e3419ae97..f28953a4a 100644 --- a/spectral_cube/tests/test_projection.py +++ b/spectral_cube/tests/test_projection.py @@ -137,8 +137,7 @@ def test_isnan(LDO, data): def test_self_arith(LDO, data): image = data - p = LDO(image, copy=False) - p.wcs = WCS() + p = LDO(image, copy=False, wcs=WCS(naxis=image.ndim)) assert hasattr(p, '_wcs') assert p.wcs is not None @@ -162,9 +161,8 @@ def test_self_arith_with_beam(LDO, data): exp_beam = Beam(1.0 * u.arcsec) image = data - p = LDO(image, copy=False) + p = LDO(image, copy=False, wcs=WCS(naxis=image.ndim)) p = p.with_beam(exp_beam) - p.wcs = WCS() assert hasattr(p, 'beam') assert hasattr(p, '_wcs') assert p.wcs is not None @@ -752,8 +750,11 @@ def test_1d_slice_round(data_255_delta, use_dask): assert hasattr(sp, '_fill_value') assert hasattr(sp.round(), '_fill_value') - assert 'OneDSpectrum' in sp.round().__repr__() - assert 'OneDSpectrum' in sp[1:-1].round().__repr__() + rnd = sp.round() + assert 'OneDSpectrum' in rnd.__repr__() + + rndslc = sp[1:-1].round() + assert 'OneDSpectrum' in rndslc.__repr__() def test_LDO_arithmetic(data_vda, use_dask): diff --git a/spectral_cube/tests/test_spectral_cube.py b/spectral_cube/tests/test_spectral_cube.py index 8f781abeb..9ebeb7df0 100644 --- a/spectral_cube/tests/test_spectral_cube.py +++ b/spectral_cube/tests/test_spectral_cube.py @@ -362,9 +362,10 @@ def test_apply_everywhere_plusminus(self, operation, value, data_advs, use_dask) ((operator.div if hasattr(operator,'div') else operator.floordiv, 0.5*u.K),)) def test_apply_everywhere_floordivide(self, operation, value, data_advs, use_dask): c1, d1 = cube_and_raw(data_advs, use_dask=use_dask) - # floordiv doesn't work, which is why it's NotImplemented - with pytest.raises(u.UnitConversionError): + try: c1o = c1._apply_everywhere(operation, value) + except Exception as ex: + isinstance(ex, (NotImplementedError, TypeError, u.UnitConversionError)) @pytest.mark.parametrize(('filename', 'trans'), translist, indirect=['filename']) diff --git a/tox.ini b/tox.ini index 159c081f8..bbd205fb8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{37,38,39,310,311}-test{,-all,-dev,-novis,-cov,-noopenfiles} + py{38,39,310,311}-test{,-dev,-noviz,-viz,-viz_extra,-cov,-noopenfiles} build_docs codestyle requires = @@ -14,11 +14,6 @@ passenv = HOME,DISPLAY,LC_ALL,LC_CTYPE,ON_TRAVIS changedir = .tmp/{envname} description = run tests with pytest deps = - dev: git+https://github.com/radio-astro-tools/pvextractor#egg=pvextractor - dev: git+https://github.com/radio-astro-tools/radio-beam#egg=radio-beam - dev: git+https://github.com/astropy/astropy#egg=astropy - dev: git+https://github.com/astropy/reproject#egg=reproject - dev: git+https://github.com/astropy/regions#egg=regions casa: numpy casa: scipy casa: matplotlib @@ -26,7 +21,12 @@ deps = casa: casatasks extras = test - all: all + dev: dev + viz: viz + viz_extra: viz_extra + noviz: noviz + cov: cov + latest: latest commands = pip freeze !cov-!noopenfiles: pytest --pyargs spectral_cube {toxinidir}/docs {posargs} From 6951e56a3deaa3991616626e8ea39a85d11db423 Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 12:30:30 -0400 Subject: [PATCH 3/5] revert to my branch, even though it was merged, to see if that can make tests go green again --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 34f944c94..3e80b5e09 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,7 +50,8 @@ dev = pvextractor @ git+https://github.com/radio-astro-tools/pvextractor#egg=pvextractor radio-beam @ git+https://github.com/radio-astro-tools/radio-beam#egg=radio-beam astropy @ git+https://github.com/astropy/astropy#egg=astropy - reproject @ git+https://github.com/astropy/reproject#egg=reproject + #reproject @ git+https://github.com/astropy/reproject#egg=reproject + reproject @ git+https://github.com/keflavich/python-reprojection.git@high_level_wcs#egg=reproject regions @ git+https://github.com/astropy/regions#egg=regions [options.package_data] From ef841c0e81c803344cf8543a1af98868e08bac8e Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 13:46:57 -0400 Subject: [PATCH 4/5] try using astropy nightly wheels. This might inadvertently make tests go green for <24h b/c the breaking change is <8h old, but the intent is to make builds faster change order of commands oh, I just had stupidly pasted in setup.cfg too --- setup.cfg | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3e80b5e09..433de34e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,7 +49,7 @@ viz_extra = dev = pvextractor @ git+https://github.com/radio-astro-tools/pvextractor#egg=pvextractor radio-beam @ git+https://github.com/radio-astro-tools/radio-beam#egg=radio-beam - astropy @ git+https://github.com/astropy/astropy#egg=astropy + # installed by tox astropy @ git+https://github.com/astropy/astropy#egg=astropy #reproject @ git+https://github.com/astropy/reproject#egg=reproject reproject @ git+https://github.com/keflavich/python-reprojection.git@high_level_wcs#egg=reproject regions @ git+https://github.com/astropy/regions#egg=regions diff --git a/tox.ini b/tox.ini index bbd205fb8..359355316 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,7 @@ extras = cov: cov latest: latest commands = + dev: pip install -U -i https://pypi.anaconda.org/astropy/simple astropy --pre pip freeze !cov-!noopenfiles: pytest --pyargs spectral_cube {toxinidir}/docs {posargs} noopenfiles: pytest --pyargs spectral_cube {toxinidir}/docs {posargs} From d0b15f9fd097958de8168635b6211115dcf027ed Mon Sep 17 00:00:00 2001 From: "Adam Ginsburg (keflavich)" Date: Mon, 13 Mar 2023 15:19:13 -0400 Subject: [PATCH 5/5] put some sticky tape over the check engine light. --- spectral_cube/tests/test_regrid.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/spectral_cube/tests/test_regrid.py b/spectral_cube/tests/test_regrid.py index 13c833522..496ddd57c 100644 --- a/spectral_cube/tests/test_regrid.py +++ b/spectral_cube/tests/test_regrid.py @@ -129,11 +129,16 @@ def test_reproject(use_memmap, data_adv, use_dask): result = cube.reproject(header_out, use_memmap=use_memmap) assert result.shape == (cube.shape[0], 5, 4) + + # empirically, this is how close we can get after https://github.com/astropy/astropy/pull/14508 + tolerance = 1e-12 + + assert wcs_out.wcs.compare(WCS(header_out).wcs, tolerance=tolerance) # Check WCS in reprojected matches wcs_out - assert wcs_out.wcs.compare(result.wcs.wcs) + assert wcs_out.wcs.compare(result.wcs.wcs, tolerance=tolerance) # And that the headers have equivalent WCS info. result_wcs_from_header = WCS(result.header) - assert result_wcs_from_header.wcs.compare(wcs_out.wcs) + assert result_wcs_from_header.wcs.compare(wcs_out.wcs, tolerance=tolerance) def test_spectral_smooth(data_522_delta, use_dask):