From 46b27f5cc9314da268119dee2b2d15115463439e Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 15 Nov 2017 15:51:36 -0800 Subject: [PATCH 01/12] fix setGrid --- Lib/hgrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/hgrid.py b/Lib/hgrid.py index 9666c5ec..7b5a504e 100644 --- a/Lib/hgrid.py +++ b/Lib/hgrid.py @@ -676,7 +676,7 @@ def isClose(self, g): def checkAxes(self, axes): """Return 1 iff every element of self.getAxisList() is in the list 'axes'.""" for item in self.getAxisList(): - if item not in axes: + if item.shape not in [axis.shape for axis in axes]: result = 0 break else: From 197aa4697aecbad818ee325b58d9c75692d74861 Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 15 Nov 2017 16:50:23 -0800 Subject: [PATCH 02/12] add squeeze array to MV2 --- Lib/MV2.py | 8 ++++++++ Lib/avariable.py | 3 +++ Lib/tvariable.py | 2 ++ 3 files changed, 13 insertions(+) diff --git a/Lib/MV2.py b/Lib/MV2.py index 16ee25a6..cb347ef4 100644 --- a/Lib/MV2.py +++ b/Lib/MV2.py @@ -340,6 +340,14 @@ def _conv_axis_arg(axis): axis = 0 return axis +def squeeze(x): + "squeze array" + ta = _makeMaskedArg(x) + maresult = numpy.squeeze(x._data) + axes, attributes, id, grid = _extractMetadata(x) + return TransientVariable( + maresult, axes=axes, attributes=attributes, grid=grid, id=id) + def is_masked(x): "Is x a 0-D masked value?" diff --git a/Lib/avariable.py b/Lib/avariable.py index ed99af63..5337d04d 100644 --- a/Lib/avariable.py +++ b/Lib/avariable.py @@ -1497,6 +1497,9 @@ def getGridIndices(self): # numpy.ma overrides + def squeeze(self): + return(MV.squeeze(self)) + def __getitem__(self, key): if isinstance(key, tuple): speclist = self._process_specs(key, {}) diff --git a/Lib/tvariable.py b/Lib/tvariable.py index e92b4ffc..0f4e5ada 100644 --- a/Lib/tvariable.py +++ b/Lib/tvariable.py @@ -121,6 +121,8 @@ def __array_finalize__(self, obj): def __copy__(self): return numpy.ma.MaskedArray.copy(self) + squeeze = AbstractVariable.squeeze + __mul__ = AbstractVariable.__mul__ __rmul__ = AbstractVariable.__rmul__ __imul__ = AbstractVariable.__imul__ From 0c992f47669ba302df305268ec7118a5b41b9f60 Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 15 Nov 2017 17:07:01 -0800 Subject: [PATCH 03/12] pinned libnetcdf=4.4.1.1=9 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4de057c1..50a3e878 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ before_install: script: #- conda install -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 - conda create -n py3 python=3.6 -- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient +- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient libnetcdf=4.4.1.1=9 - source activate py3 - if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy=7.1.0.dev34; fi - if [ "$TRAVIS_OS_NAME" = "osx" ]; then conda install -n py3 -c nadeau1 -c conda-forge esmf esmpy; fi From dddef7bce324606978669efe3e1ae60edd77f60d Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 15 Nov 2017 17:09:53 -0800 Subject: [PATCH 04/12] pinned libnetcdf=4.4.1.1=9 for MAC --- ci-support/circleci_mac_dep.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-support/circleci_mac_dep.sh b/ci-support/circleci_mac_dep.sh index 7f360a3a..cddce580 100644 --- a/ci-support/circleci_mac_dep.sh +++ b/ci-support/circleci_mac_dep.sh @@ -12,7 +12,7 @@ echo $cmd $cmd # Create Python 3 environment -cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient" +cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient libnetcdf=4.4.1.1=9" echo $cmd $cmd From e7b1846fb7a965e643a0dbbf1af1ad64a23e9c38 Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 15 Nov 2017 17:17:17 -0800 Subject: [PATCH 05/12] fix documentation for squeeze --- Lib/MV2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/MV2.py b/Lib/MV2.py index cb347ef4..f0856e7e 100644 --- a/Lib/MV2.py +++ b/Lib/MV2.py @@ -341,7 +341,7 @@ def _conv_axis_arg(axis): return axis def squeeze(x): - "squeze array" + "call numpy.squeeze on ndarray and rebuild tvariable." ta = _makeMaskedArg(x) maresult = numpy.squeeze(x._data) axes, attributes, id, grid = _extractMetadata(x) From 6eb5819bde7b7bb5281239118f4bd42bfce269a9 Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 15 Nov 2017 17:28:37 -0800 Subject: [PATCH 06/12] fix ci machines --- .travis.yml | 6 +++--- ci-support/circleci_mac_dep.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 50a3e878..3404d87f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,9 +20,9 @@ before_install: script: #- conda install -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 - conda create -n py3 python=3.6 -- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient libnetcdf=4.4.1.1=9 +- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient libnetcdf netcdf-fortran=4.4.4=3 - source activate py3 -- if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy=7.1.0.dev34; fi +- if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy=7.1.0.dev34 netcdf-fortran=4.4.4=3; fi - if [ "$TRAVIS_OS_NAME" = "osx" ]; then conda install -n py3 -c nadeau1 -c conda-forge esmf esmpy; fi - export UVCDAT_ANONYMOUS_LOG=False - echo $TRAVIS_BRANCH @@ -38,7 +38,7 @@ script: - conda create -n py2 python=2.7 - source activate py2 - which python -- conda install -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 +- conda install -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 netcdf-fortran=4.4.4=3 - export UVCDAT_ANONYMOUS_LOG=False - python setup.py install - python run_tests.py -v2 -s diff --git a/ci-support/circleci_mac_dep.sh b/ci-support/circleci_mac_dep.sh index cddce580..877f8d08 100644 --- a/ci-support/circleci_mac_dep.sh +++ b/ci-support/circleci_mac_dep.sh @@ -12,11 +12,11 @@ echo $cmd $cmd # Create Python 3 environment -cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient libnetcdf=4.4.1.1=9" +cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient libnetcdf netcdf-fortran=4.4.4=3" echo $cmd $cmd -cmd="conda install -n py3 -c nadeau1 -c conda-forge esmf esmpy" +cmd="conda install -n py3 -c nadeau1 -c conda-forge esmf esmpy netcdf-fortran=4.4.4=3" echo $cmd $cmd @@ -30,7 +30,7 @@ cmd="source activate py2" echo $cmd $cmd -cmd="conda install -n py2 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient" +cmd="conda install -n py2 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=3" echo $cmd $cmd From d572063d605cc2a7ac298980a3f643abc3d1dd91 Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 15 Nov 2017 17:48:50 -0800 Subject: [PATCH 07/12] flake8 --- Lib/MV2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/MV2.py b/Lib/MV2.py index f0856e7e..4da7e1d7 100644 --- a/Lib/MV2.py +++ b/Lib/MV2.py @@ -5,7 +5,7 @@ import numpy from numpy import character, float, float32, float64 # noqa from numpy import int, int8, int16, int32, int64, byte # noqa -from numpy import ubyte, uint8, uint16, uint32, uint64, long # noqa +from numpy import ubyte, uint8, uint16, uint32, uint64, long # noqa from numpy.ma import allclose, allequal, common_fill_value # noqa from numpy.ma import make_mask_none, dot, filled # noqa from numpy.ma import getmask, getmaskarray, identity # noqa @@ -340,13 +340,14 @@ def _conv_axis_arg(axis): axis = 0 return axis + def squeeze(x): "call numpy.squeeze on ndarray and rebuild tvariable." - ta = _makeMaskedArg(x) + # ta = _makeMaskedArg(x) maresult = numpy.squeeze(x._data) axes, attributes, id, grid = _extractMetadata(x) return TransientVariable( - maresult, axes=axes, attributes=attributes, grid=grid, id=id) + maresult, axes=axes, attributes=attributes, grid=grid, id=id) def is_masked(x): From 4c14ce620342b43a800064a98ca5d32920dabc4c Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 29 Nov 2017 18:42:15 -0800 Subject: [PATCH 08/12] fix squeeze UV-CDAT/uvcdat/issues/1707 --- Lib/hgrid.py | 21 ++++++++++++--------- tests/test_curvilinear_grid.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Lib/hgrid.py b/Lib/hgrid.py index f07684ab..d163b9c9 100644 --- a/Lib/hgrid.py +++ b/Lib/hgrid.py @@ -588,14 +588,16 @@ def getGridSlices(self, domainlist, newaxislist, slicelist): k = 0 i = j = -1 for d in domainlist: - if d is iaxis: - inewaxis = newaxislist[k] - islice = slicelist[k] - i = k - if d is jaxis: - jnewaxis = newaxislist[k] - jslice = slicelist[k] - j = k + if d.shape == iaxis.shape: + if numpy.allclose(d[:], iaxis[:]) == True: + inewaxis = newaxislist[k] + islice = slicelist[k] + i = k + if d.shape == jaxis.shape: + if numpy.allclose(d[:], jaxis[:]) == True: + jnewaxis = newaxislist[k] + jslice = slicelist[k] + j = k k += 1 if i == -1 or j == -1: @@ -677,7 +679,8 @@ def isClose(self, g): def checkAxes(self, axes): """Return 1 iff every element of self.getAxisList() is in the list 'axes'.""" for item in self.getAxisList(): - if item.shape not in [axis.shape for axis in axes]: + # if all [False, False, ....] result=0 + if not any([allclose(item[:],axis[:]) for axis in axes]): result = 0 break else: diff --git a/tests/test_curvilinear_grid.py b/tests/test_curvilinear_grid.py index 0a2a77eb..e3fc8ce3 100755 --- a/tests/test_curvilinear_grid.py +++ b/tests/test_curvilinear_grid.py @@ -5,9 +5,19 @@ import os import sys import basetest +import copy class TestCurvilinearGrids(basetest.CDMSBaseTest): + + def testReadCurvGrid(self): + f = self.getDataFile('sampleCurveGrid4.nc') + data = f("sample") + attrs = copy.copy(data.attributes) + axes = list([x[0].clone() for x in data.getDomain()]) + data2 = cdms2.createVariable(data, copy=0, attributes=attrs, axes=axes) + data3=data2(*(),squeeze=1) + def testCurvilinear(self): datb = numpy.array( [[697., 698., 699., 700., 701., 702., ], From 2c7bec773139c121e92cc40d2e129b2df21ebb4a Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 29 Nov 2017 18:43:09 -0800 Subject: [PATCH 09/12] fix squeeze UV-CDAT/uvcdat/issues/1707 --- .travis.yml | 14 -------------- ci-support/circleci_mac_dep.sh | 6 ------ 2 files changed, 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8d691f93..5cccc2db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,23 +20,9 @@ before_install: script: #- conda install -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 - conda create -n py3 python=3.6 -<<<<<<< HEAD -- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient libnetcdf netcdf-fortran=4.4.4=3 -||||||| merged common ancestors -- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient -======= - conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=3 ->>>>>>> master - source activate py3 -<<<<<<< HEAD - if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy=7.1.0.dev34 netcdf-fortran=4.4.4=3; fi -- if [ "$TRAVIS_OS_NAME" = "osx" ]; then conda install -n py3 -c nadeau1 -c conda-forge esmf esmpy; fi -||||||| merged common ancestors -- if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy=7.1.0.dev34; fi -- if [ "$TRAVIS_OS_NAME" = "osx" ]; then conda install -n py3 -c nadeau1 -c conda-forge esmf esmpy; fi -======= -- if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy=7.1.0.dev34 netcdf-fortran=4.4.4=3; fi ->>>>>>> master - export UVCDAT_ANONYMOUS_LOG=False - echo $TRAVIS_BRANCH - export TRAVIS_PR_BRANCH=$TRAVIS_BRANCH diff --git a/ci-support/circleci_mac_dep.sh b/ci-support/circleci_mac_dep.sh index 6a45f418..65994566 100644 --- a/ci-support/circleci_mac_dep.sh +++ b/ci-support/circleci_mac_dep.sh @@ -12,13 +12,7 @@ echo $cmd $cmd # Create Python 3 environment -<<<<<<< HEAD -cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient libnetcdf netcdf-fortran=4.4.4=3" -||||||| merged common ancestors -cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient" -======= cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=3" ->>>>>>> master echo $cmd $cmd From 2b2f79e7e28c6ba3dd53d99a04352ff3260ec35c Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 29 Nov 2017 18:51:44 -0800 Subject: [PATCH 10/12] fix ci netcdf-fortran --- .travis.yml | 2 +- ci-support/circleci_mac_dep.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5cccc2db..cd78a901 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ before_install: script: #- conda install -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 - conda create -n py3 python=3.6 -- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=3 +- conda install -n py3 -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=5 - source activate py3 - if [ "$TRAVIS_OS_NAME" = "linux" ]; then conda install -n py3 -c nesii/channel/dev-esmf -c conda-forge esmpy=7.1.0.dev34 netcdf-fortran=4.4.4=3; fi - export UVCDAT_ANONYMOUS_LOG=False diff --git a/ci-support/circleci_mac_dep.sh b/ci-support/circleci_mac_dep.sh index 65994566..08f2f2ea 100644 --- a/ci-support/circleci_mac_dep.sh +++ b/ci-support/circleci_mac_dep.sh @@ -12,7 +12,7 @@ echo $cmd $cmd # Create Python 3 environment -cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=3" +cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=5" echo $cmd $cmd From 7365a0a1c1162a07412150c40a4bd735b08142d7 Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 29 Nov 2017 18:56:06 -0800 Subject: [PATCH 11/12] fix ci netcdf-fortran to number 4.4.4=5 --- ci-support/circleci_mac_dep.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-support/circleci_mac_dep.sh b/ci-support/circleci_mac_dep.sh index 08f2f2ea..d7a7661e 100644 --- a/ci-support/circleci_mac_dep.sh +++ b/ci-support/circleci_mac_dep.sh @@ -16,7 +16,7 @@ cmd="conda create -n py3 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf echo $cmd $cmd -cmd="conda install -n py3 -c nadeau1 -c conda-forge esmf esmpy netcdf-fortran=4.4.4=3" +cmd="conda install -n py3 -c nadeau1 -c conda-forge esmf esmpy netcdf-fortran=4.4.4=5" echo $cmd $cmd @@ -30,7 +30,7 @@ cmd="source activate py2" echo $cmd $cmd -cmd="conda install -n py2 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=3" +cmd="conda install -n py2 -c uvcdat/label/nightly -c conda-forge -c uvcdat libcf distarray cdtime libcdms cdat_info numpy esmf esmpy libdrs_f pyopenssl nose requests flake8 myproxyclient netcdf-fortran=4.4.4=5" echo $cmd $cmd From eacc870becf7a324b2e8474c1dd84d9016bdfed3 Mon Sep 17 00:00:00 2001 From: Denis Nadeau Date: Wed, 29 Nov 2017 19:01:19 -0800 Subject: [PATCH 12/12] flake8 --- Lib/hgrid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/hgrid.py b/Lib/hgrid.py index d163b9c9..18d7e0a2 100644 --- a/Lib/hgrid.py +++ b/Lib/hgrid.py @@ -589,12 +589,12 @@ def getGridSlices(self, domainlist, newaxislist, slicelist): i = j = -1 for d in domainlist: if d.shape == iaxis.shape: - if numpy.allclose(d[:], iaxis[:]) == True: + if numpy.allclose(d[:], iaxis[:]) is True: inewaxis = newaxislist[k] islice = slicelist[k] i = k if d.shape == jaxis.shape: - if numpy.allclose(d[:], jaxis[:]) == True: + if numpy.allclose(d[:], jaxis[:]) is True: jnewaxis = newaxislist[k] jslice = slicelist[k] j = k @@ -680,7 +680,7 @@ def checkAxes(self, axes): """Return 1 iff every element of self.getAxisList() is in the list 'axes'.""" for item in self.getAxisList(): # if all [False, False, ....] result=0 - if not any([allclose(item[:],axis[:]) for axis in axes]): + if not any([allclose(item[:], axis[:]) for axis in axes]): result = 0 break else: