From 38c0e990aa2b97f4296e0a652eb855a0af0bb1de Mon Sep 17 00:00:00 2001 From: Duncan Watson-Parris Date: Thu, 24 May 2018 22:15:17 +0100 Subject: [PATCH] Squashed commit of the following: commit 661943b9ed7f65f65a26e099e194e1b4ecc78036 Merge: 54f1937 40be11b Author: Duncan Watson-Parris Date: Thu May 24 22:12:45 2018 +0100 Merge-up master # Conflicts: # lib/iris/coords.py commit 54f1937704e39ae54ccbbb6f14a26d8e3ae2777f Author: Duncan Watson-Parris Date: Thu May 24 21:30:37 2018 +0100 Reverting test data commit 4588a29a51ae017ba93ff774fe4b1088ad938c06 Author: Duncan Watson-Parris Date: Fri May 4 10:07:37 2018 +0100 Fix concatenation (cherry picked from commit 3e647ac) commit 97c2c609b072bf00b5395de3fedb978f665510af Author: Duncan Watson-Parris Date: Wed May 2 10:23:22 2018 +0100 Update test results - the points are now the mean rather than the midpoint, but the bounds don't change commit ffed4211c310da39eec65972a151a909650609e7 Author: Duncan Watson-Parris Date: Tue May 1 16:20:35 2018 +0100 Fixing long lines commit e9f7b5c70ffea9ce85dfd95e9d0233d5b173f8ee Author: Duncan Watson-Parris Date: Tue May 1 16:09:08 2018 +0100 Update the test points to reflect the mean of the original points, rather than midpoint of the bounds commit cc85aea542127abdad84fa12dbab744f4d219907 Author: Duncan Watson-Parris Date: Tue May 1 14:27:14 2018 +0100 Adding support for partial collapse of multi-dimensional coordinates commit cb41ddb0de793c4627542a6c03ab1335848658a3 Author: Duncan Watson-Parris Date: Thu May 24 21:30:37 2018 +0100 Reverting test data commit 7e6ab38ff939273b50e825427e0ff01b30cd13f4 Author: Duncan Watson-Parris Date: Thu May 24 21:18:18 2018 +0100 Revert to calculating the points as the mid-points of the new bounds (as per the existing behaviour). Removing lazy points test since they now just inherit the 'laziness' of the bounds. commit b147a426f5db6fbeab2d81ec1e9419b3253c6f7c Author: Duncan Watson-Parris Date: Fri May 4 10:07:37 2018 +0100 Fix concatenation (cherry picked from commit 3e647ac) commit fd5add67795f47fe83b819d169db0ef2706dccfb Author: Duncan Watson-Parris Date: Thu May 3 21:45:35 2018 +0100 Fix PEP style ( though I prefer the previous indentation...) (cherry picked from commit 5bce7bf) commit 2e31ba30ccd943c9e766412963090a4fa7c2d3cf Author: Duncan Watson-Parris Date: Thu May 3 21:40:26 2018 +0100 Reinstating support for (partial) collapse of lazy coordinate points / bounds. Added some tests to catch this in the future. (cherry picked from commit 386edc7) commit 20834b0c0ae3568a84374cac105a3c57ea12107f Author: Duncan Watson-Parris Date: Thu May 3 21:38:51 2018 +0100 Adding whatsnew (cherry picked from commit 9bce3ab) commit 00072f049032fa3610c4823d71208be389731c55 Merge: 8851a65 b15f4ed Author: Duncan Watson-Parris Date: Thu May 3 20:03:30 2018 +0100 Merge branch 'master' of https://github.com/scitools/iris commit 8851a65c8d98575aa3f0148c23d7859bb0d057aa Author: Duncan Watson-Parris Date: Wed May 2 10:23:22 2018 +0100 Update test results - the points are now the mean rather than the midpoint, but the bounds don't change commit f54ae953d8fb9edf895563c3a83b752c821ad019 Author: Duncan Watson-Parris Date: Wed May 2 10:16:26 2018 +0100 Update headers commit 31e9ef312edf8027a3de888a6ddcf3cc1502bb71 Author: Duncan Watson-Parris Date: Tue May 1 16:28:16 2018 +0100 Fixing long lines commit 861f7c5fc537b29e511e7664680f13e6c0084a05 Author: Duncan Watson-Parris Date: Tue May 1 16:20:35 2018 +0100 Fixing long lines commit 2ad0b5c0279087eaabead78d09a4e80273c9c2ee Author: Duncan Watson-Parris Date: Tue May 1 16:09:08 2018 +0100 Update the test points to reflect the mean of the original points, rather than midpoint of the bounds commit ffd0db2e1fe8d33f89bdbb08451b6adf0f71ecb5 Merge: 6a50076 0d6d026 Author: Duncan Watson-Parris Date: Tue May 1 16:02:02 2018 +0100 Merge branch 'master' of https://github.com/scitools/iris commit 6a50076776ad4d8998429831c8b7881a24654419 Author: Duncan Watson-Parris Date: Tue May 1 14:27:14 2018 +0100 Adding support for partial collapse of multi-dimensional coordinates --- ...wfeature_2018-May-03_multidim_collapse.txt | 2 + lib/iris/coords.py | 48 +++++--------- lib/iris/cube.py | 16 ----- lib/iris/tests/test_analysis.py | 50 ++++++++++++++- lib/iris/tests/unit/coords/test_Coord.py | 64 +++++++++++++++---- 5 files changed, 120 insertions(+), 60 deletions(-) create mode 100644 docs/iris/src/whatsnew/contributions_2.1/newfeature_2018-May-03_multidim_collapse.txt diff --git a/docs/iris/src/whatsnew/contributions_2.1/newfeature_2018-May-03_multidim_collapse.txt b/docs/iris/src/whatsnew/contributions_2.1/newfeature_2018-May-03_multidim_collapse.txt new file mode 100644 index 0000000000..e9dc217cb3 --- /dev/null +++ b/docs/iris/src/whatsnew/contributions_2.1/newfeature_2018-May-03_multidim_collapse.txt @@ -0,0 +1,2 @@ +* The partial collapse of multi-dimensional auxiliary coordinates is now + supported. Collapsed bounds span the range of the collapsed dimension(s). \ No newline at end of file diff --git a/lib/iris/coords.py b/lib/iris/coords.py index b9ba85a46c..3b2b2f5eb1 100644 --- a/lib/iris/coords.py +++ b/lib/iris/coords.py @@ -1169,20 +1169,14 @@ def collapsed(self, dims_to_collapse=None): the specified dimensions. Replaces the points & bounds with a simple bounded region. - - .. note:: - You cannot partially collapse a multi-dimensional coordinate. See - :ref:`cube.collapsed ` for more - information. - """ + import dask.array as da + # Ensure dims_to_collapse is a tuple to be able to pass + # through to numpy if isinstance(dims_to_collapse, (int, np.integer)): - dims_to_collapse = [dims_to_collapse] - - if dims_to_collapse is not None and \ - set(range(self.ndim)) != set(dims_to_collapse): - raise ValueError('Cannot partially collapse a coordinate (%s).' - % self.name()) + dims_to_collapse = (dims_to_collapse, ) + if isinstance(dims_to_collapse, list): + dims_to_collapse = tuple(dims_to_collapse) if np.issubdtype(self.dtype, np.str_): # Collapse the coordinate by serializing the points and @@ -1215,28 +1209,20 @@ def serialize(x): 'Metadata may not be fully descriptive for {!r}.' warnings.warn(msg.format(self.name())) - # Create bounds for the new collapsed coordinate. - item = self.core_bounds() if self.has_bounds() \ + # Determine the array library for stacking + al = da if self.has_bounds() \ + and _lazy.is_lazy_data(self.core_bounds()) else np + + item = al.concatenate(self.core_bounds()) if self.has_bounds() \ else self.core_points() - lower, upper = item.min(), item.max() - bounds_dtype = item.dtype - # Ensure 2D shape of new bounds. - bounds = np.empty((1, 2), 'object') - bounds[0, 0] = lower - bounds[0, 1] = upper - # Create points for the new collapsed coordinate. - points_dtype = self.dtype - points = (float(lower) + float(upper)) * 0.5 + + # Calculate the bounds and points along the right dims + bounds = al.stack([item.min(axis=dims_to_collapse), + item.max(axis=dims_to_collapse)]).T + points = al.array(bounds.sum(axis=-1) * 0.5, dtype=self.dtype) # Create the new collapsed coordinate. - if _lazy.is_lazy_data(item): - bounds = _lazy.multidim_lazy_stack(bounds) - coord = self.copy(points=points, bounds=bounds) - else: - bounds = np.concatenate(bounds) - bounds = np.array(bounds, dtype=bounds_dtype) - coord = self.copy(points=np.array(points, dtype=points_dtype), - bounds=bounds) + coord = self.copy(points=points, bounds=bounds) return coord def _guess_bounds(self, bound_position=0.5): diff --git a/lib/iris/cube.py b/lib/iris/cube.py index b7066d9258..54517820cd 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -3155,22 +3155,6 @@ def collapsed(self, coords, aggregator, **kwargs): cube.collapsed(['latitude', 'longitude'], iris.analysis.VARIANCE) - - .. _partially_collapse_multi-dim_coord: - - .. note:: - You cannot partially collapse a multi-dimensional coordinate. Doing - so would result in a partial collapse of the multi-dimensional - coordinate. Instead you must either: - * collapse in a single operation all cube axes that the - multi-dimensional coordinate spans, - * remove the multi-dimensional coordinate from the cube before - performing the collapse operation, or - * not collapse the coordinate at all. - - Multi-dimensional derived coordinates will not prevent a successful - collapse operation. - """ # Convert any coordinate names to coordinates coords = self._as_list_of_coords(coords) diff --git a/lib/iris/tests/test_analysis.py b/lib/iris/tests/test_analysis.py index dc8ff1985a..e07848a3d8 100644 --- a/lib/iris/tests/test_analysis.py +++ b/lib/iris/tests/test_analysis.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2017, Met Office +# (C) British Crown Copyright 2010 - 2018, Met Office # # This file is part of Iris. # @@ -310,6 +310,54 @@ def test_sum(self): np.testing.assert_array_equal(cube.data, np.array([6, 18, 17])) +class TestAuxCoordCollapse(tests.IrisTest): + + def setUp(self): + self.cube_with_aux_coord = tests.stock.simple_4d_with_hybrid_height() + + # Guess bounds to get the weights + self.cube_with_aux_coord.coord('grid_latitude').guess_bounds() + self.cube_with_aux_coord.coord('grid_longitude').guess_bounds() + + def test_max(self): + cube = self.cube_with_aux_coord.collapsed('grid_latitude', + iris.analysis.MAX) + np.testing.assert_array_equal(cube.coord('surface_altitude').points, + np.array([112, 113, 114, + 115, 116, 117])) + + np.testing.assert_array_equal(cube.coord('surface_altitude').bounds, + np.array([[100, 124], + [101, 125], + [102, 126], + [103, 127], + [104, 128], + [105, 129]])) + + # Check collapsing over the whole coord still works + cube = self.cube_with_aux_coord.collapsed('altitude', + iris.analysis.MAX) + + np.testing.assert_array_equal(cube.coord('surface_altitude').points, + np.array([114])) + + np.testing.assert_array_equal(cube.coord('surface_altitude').bounds, + np.array([[100, 129]])) + + cube = self.cube_with_aux_coord.collapsed('grid_longitude', + iris.analysis.MAX) + + np.testing.assert_array_equal(cube.coord('surface_altitude').points, + np.array([102, 108, 114, 120, 126])) + + np.testing.assert_array_equal(cube.coord('surface_altitude').bounds, + np.array([[100, 105], + [106, 111], + [112, 117], + [118, 123], + [124, 129]])) + + class TestAggregator_mdtol_keyword(tests.IrisTest): def setUp(self): data = ma.array([[1, 2], [4, 5]], dtype=np.float32, diff --git a/lib/iris/tests/unit/coords/test_Coord.py b/lib/iris/tests/unit/coords/test_Coord.py index 78d0b82486..b43df9447a 100644 --- a/lib/iris/tests/unit/coords/test_Coord.py +++ b/lib/iris/tests/unit/coords/test_Coord.py @@ -31,6 +31,7 @@ from iris.coords import DimCoord, AuxCoord, Coord from iris.tests import mock from iris.exceptions import UnitConversionError +from iris.tests.unit.coords import CoordTestMixin Pair = collections.namedtuple('Pair', 'points bounds') @@ -232,7 +233,7 @@ def test_time_as_object(self): mock.sentinel.upper))]) -class Test_collapsed(tests.IrisTest): +class Test_collapsed(tests.IrisTest, CoordTestMixin): def test_serialize(self): # Collapse a string AuxCoord, causing it to be serialised. @@ -277,19 +278,58 @@ def test_dim_1d(self): [[coord.bounds.min(), coord.bounds.max()]]) def test_numeric_nd(self): - # Contiguous only defined for 2d bounds. - coord = AuxCoord(points=np.array([3, 6, 9]), - bounds=np.array([[1, 2, 4, 5], + coord = AuxCoord(points=np.array([[1, 2, 4, 5], [4, 5, 7, 8], [7, 8, 10, 11]])) - with self.assertRaises(ValueError): - coord.collapsed() - - def test_collapsed_overflow(self): - coord = DimCoord(points=np.array([1493892000, 1493895600, 1493899200], - dtype=np.int32)) - result = coord.collapsed() - self.assertEqual(result.points, 1493895600) + + collapsed_coord = coord.collapsed() + self.assertArrayEqual(collapsed_coord.points, np.array([6])) + self.assertArrayEqual(collapsed_coord.bounds, np.array([[1, 11]])) + + # Test partially collapsing one dimension... + collapsed_coord = coord.collapsed(1) + self.assertArrayEqual(collapsed_coord.points, np.array([3., 6., 9.])) + self.assertArrayEqual(collapsed_coord.bounds, np.array([[1, 5], + [4, 8], + [7, 11]])) + + # ... and the other + collapsed_coord = coord.collapsed(0) + self.assertArrayEqual(collapsed_coord.points, np.array([4, 5, 7, 8])) + self.assertArrayEqual(collapsed_coord.bounds, np.array([[1, 7], + [2, 8], + [4, 10], + [5, 11]])) + + def test_lazy_nd_bounds(self): + import dask.array as da + + self.setupTestArrays((3, 4)) + coord = AuxCoord(self.pts_real, bounds=self.bds_lazy) + + collapsed_coord = coord.collapsed() + + # Note that the new points get recalculated from the lazy bounds + # and so end up as lazy + self.assertTrue(collapsed_coord.has_lazy_points()) + self.assertTrue(collapsed_coord.has_lazy_bounds()) + + self.assertArrayEqual(collapsed_coord.points, np.array([55])) + self.assertArrayEqual(collapsed_coord.bounds, da.array([[-2, 112]])) + + def test_lazy_nd_points_and_bounds(self): + import dask.array as da + + self.setupTestArrays((3, 4)) + coord = AuxCoord(self.pts_lazy, bounds=self.bds_lazy) + + collapsed_coord = coord.collapsed() + + self.assertTrue(collapsed_coord.has_lazy_points()) + self.assertTrue(collapsed_coord.has_lazy_bounds()) + + self.assertArrayEqual(collapsed_coord.points, da.array([55])) + self.assertArrayEqual(collapsed_coord.bounds, da.array([[-2, 112]])) class Test_is_compatible(tests.IrisTest):