From a2d293765693bc826cbe6ed7a2d531d6b2f9401e Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 3 Mar 2020 21:19:38 -0500 Subject: [PATCH 01/10] allow ellipsis to be used in stack --- doc/reshaping.rst | 7 +++++++ xarray/core/dataset.py | 2 ++ xarray/tests/test_dataarray.py | 3 +++ xarray/tests/test_dataset.py | 6 ++++++ 4 files changed, 18 insertions(+) diff --git a/doc/reshaping.rst b/doc/reshaping.rst index 455a24f9216..251c01b6339 100644 --- a/doc/reshaping.rst +++ b/doc/reshaping.rst @@ -109,6 +109,12 @@ implemented :py:meth:`~xarray.DataArray.stack` and stacked stacked.unstack('z') +As elsewhere in xarray, an ellipsis (`...`) can be used to represent all dimensions: + +.. ipython:: python + stacked = array.stack(z=...) + stacked + These methods are modeled on the :py:class:`pandas.DataFrame` methods of the same name, although in xarray they always create new dimensions rather than adding to the existing index or columns. @@ -164,6 +170,7 @@ like this: 'b': ('x', [6, 7])}, coords={'y': ['u', 'v', 'w']} ) + data stacked = data.to_stacked_array("z", sample_dims=['x']) stacked unstacked = stacked.to_unstacked_dataset("z") diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 7252dd2f3df..3f5cf93a03b 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -3235,6 +3235,8 @@ def reorder_levels( return self._replace(variables, indexes=indexes) def _stack_once(self, dims, new_dim): + if dims == ... or dims == [...]: + dims = self.dims variables = {} for name, var in self.variables.items(): if name not in dims: diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 0a622d279ba..8f2332f9229 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2040,6 +2040,9 @@ def test_stack_unstack(self): actual = orig.stack(z=["x", "y"]).unstack("z").drop_vars(["x", "y"]) assert_identical(orig, actual) + actual = orig.stack(z=...).unstack("z").drop_vars(["x", "y"]) + assert_identical(orig, actual) + dims = ["a", "b", "c", "d", "e"] orig = xr.DataArray(np.random.rand(1, 2, 3, 2, 1), dims=dims) stacked = orig.stack(ab=["a", "b"], cd=["c", "d"]) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 7bcf9379ae8..86bd1eb5f31 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2868,6 +2868,12 @@ def test_stack(self): actual = ds.stack(z=["x", "y"]) assert_identical(expected, actual) + actual = ds.stack(z=[...]) + assert_identical(expected, actual) + + actual = ds.stack(z=...) + assert_identical(expected, actual) + exp_index = pd.MultiIndex.from_product([["a", "b"], [0, 1]], names=["y", "x"]) expected = Dataset( {"a": ("z", [0, 1, 0, 1]), "b": ("z", [0, 2, 1, 3]), "z": exp_index} From 875c3b999c1ad639c320eb8acb2388e4b1501b51 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 3 Mar 2020 22:01:44 -0500 Subject: [PATCH 02/10] doc fix --- doc/reshaping.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/reshaping.rst b/doc/reshaping.rst index 251c01b6339..881f841ea72 100644 --- a/doc/reshaping.rst +++ b/doc/reshaping.rst @@ -112,6 +112,7 @@ implemented :py:meth:`~xarray.DataArray.stack` and As elsewhere in xarray, an ellipsis (`...`) can be used to represent all dimensions: .. ipython:: python + stacked = array.stack(z=...) stacked From 48809a0684685f3d0bb81cc632ef91fcb5b57dfa Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 4 Mar 2020 07:19:03 -0500 Subject: [PATCH 03/10] support ellipsis only as part of an iterable --- xarray/core/dataset.py | 2 +- xarray/tests/test_dataarray.py | 2 +- xarray/tests/test_dataset.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 3f5cf93a03b..51debd1e61d 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -3235,7 +3235,7 @@ def reorder_levels( return self._replace(variables, indexes=indexes) def _stack_once(self, dims, new_dim): - if dims == ... or dims == [...]: + if list(dims) == [...]: dims = self.dims variables = {} for name, var in self.variables.items(): diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 8f2332f9229..3d4bae18f94 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -2040,7 +2040,7 @@ def test_stack_unstack(self): actual = orig.stack(z=["x", "y"]).unstack("z").drop_vars(["x", "y"]) assert_identical(orig, actual) - actual = orig.stack(z=...).unstack("z").drop_vars(["x", "y"]) + actual = orig.stack(z=[...]).unstack("z").drop_vars(["x", "y"]) assert_identical(orig, actual) dims = ["a", "b", "c", "d", "e"] diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 86bd1eb5f31..d789e0c067b 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2871,7 +2871,8 @@ def test_stack(self): actual = ds.stack(z=[...]) assert_identical(expected, actual) - actual = ds.stack(z=...) + # non list dims with ellipsis + actual = ds.stack(z=(...,)) assert_identical(expected, actual) exp_index = pd.MultiIndex.from_product([["a", "b"], [0, 1]], names=["y", "x"]) From b1b3e909d2014a8a456de7c6ced97149b968dec5 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 4 Mar 2020 07:20:21 -0500 Subject: [PATCH 04/10] docs --- doc/reshaping.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/reshaping.rst b/doc/reshaping.rst index 881f841ea72..90a6efe3414 100644 --- a/doc/reshaping.rst +++ b/doc/reshaping.rst @@ -113,7 +113,7 @@ As elsewhere in xarray, an ellipsis (`...`) can be used to represent all dimensi .. ipython:: python - stacked = array.stack(z=...) + stacked = array.stack(z=[...]) stacked These methods are modeled on the :py:class:`pandas.DataFrame` methods of the From d7cac819c83ebae9474e170a4d3a514c1ec833ac Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 4 Mar 2020 07:27:23 -0500 Subject: [PATCH 05/10] whatsnew --- doc/whats-new.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 4a6083522ba..b2bd579beb6 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,14 +36,19 @@ New Features By `Justus Magin `_. - :py:meth:`Dataset.groupby` and :py:meth:`DataArray.groupby` now raise a `TypeError` on multiple string arguments. Receiving multiple string arguments - often means a user is attempting to pass multiple dimensions to group over - and should instead pass a list. + often means a user is attempting to pass multiple dimensions as seeparate + argument and should instead pass a list. + (:pull:`3802`) By `Maximilian Roos `_ - The new ``Dataset._repr_html_`` and ``DataArray._repr_html_`` (introduced in 0.14.1) is now on by default. To disable, use ``xarray.set_options(display_style="text")``. By `Julia Signell `_. - +- An ellipsis (``...``) is now supported in the ``dims`` argument of + :py:meth:`Dataset.stack` and :py:meth:`DataArray.stack` to mean all + dimensions, similar to its meaning in :py:meth:`DataArray.transpose` + (:pull:`3826`) + By `Maximilian Roos `_ Bug fixes ~~~~~~~~~ From 134db2d83ce92cac6e43680a92635f3883638de3 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 4 Mar 2020 07:36:26 -0500 Subject: [PATCH 06/10] docstring, whatsnew --- doc/whats-new.rst | 6 +++--- xarray/core/dataarray.py | 3 ++- xarray/core/dataset.py | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index b2bd579beb6..5eee190f18e 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -36,8 +36,8 @@ New Features By `Justus Magin `_. - :py:meth:`Dataset.groupby` and :py:meth:`DataArray.groupby` now raise a `TypeError` on multiple string arguments. Receiving multiple string arguments - often means a user is attempting to pass multiple dimensions as seeparate - argument and should instead pass a list. + often means a user is attempting to pass multiple dimensions as separate + arguments and should instead pass a single list of dimensions. (:pull:`3802`) By `Maximilian Roos `_ - The new ``Dataset._repr_html_`` and ``DataArray._repr_html_`` (introduced @@ -45,7 +45,7 @@ New Features ``xarray.set_options(display_style="text")``. By `Julia Signell `_. - An ellipsis (``...``) is now supported in the ``dims`` argument of - :py:meth:`Dataset.stack` and :py:meth:`DataArray.stack` to mean all + :py:meth:`Dataset.stack` and :py:meth:`DataArray.stack`, meaning all dimensions, similar to its meaning in :py:meth:`DataArray.transpose` (:pull:`3826`) By `Maximilian Roos `_ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index b1da0ca1448..0c9baefdad7 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1705,7 +1705,8 @@ def stack( ---------- dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they - replace. + replace. Passing a list with an ellipsis (`[...]`) will stack over all + dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. One of dimensions or dimensions_kwargs must be provided. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 51debd1e61d..2c0b609d796 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -3279,7 +3279,8 @@ def stack( ---------- dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they - replace. + replace. Passing a list with an ellipsis (`[...]`) will stack over all + dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. One of dimensions or dimensions_kwargs must be provided. From ae18a7aa050d698b1ed2e9a0afce4fe646b4a67d Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 4 Mar 2020 07:40:21 -0500 Subject: [PATCH 07/10] docstring, whatsnew --- doc/whats-new.rst | 2 +- xarray/core/dataarray.py | 4 ++-- xarray/core/dataset.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 5eee190f18e..9be349f0173 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -46,7 +46,7 @@ New Features By `Julia Signell `_. - An ellipsis (``...``) is now supported in the ``dims`` argument of :py:meth:`Dataset.stack` and :py:meth:`DataArray.stack`, meaning all - dimensions, similar to its meaning in :py:meth:`DataArray.transpose` + dimensions, similar to its meaning in :py:meth:`DataArray.transpose`. (:pull:`3826`) By `Maximilian Roos `_ diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 0c9baefdad7..474ab1d206d 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1705,8 +1705,8 @@ def stack( ---------- dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they - replace. Passing a list with an ellipsis (`[...]`) will stack over all - dimensions. + replace. Passing a list with an ellipsis (`[...]`) as existing + dimensions will stack over all dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. One of dimensions or dimensions_kwargs must be provided. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 2c0b609d796..00e3e45fd59 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -3279,8 +3279,8 @@ def stack( ---------- dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they - replace. Passing a list with an ellipsis (`[...]`) will stack over all - dimensions. + replace. Passing a list with an ellipsis (`[...]`) as existing + dimensions will stack over all dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. One of dimensions or dimensions_kwargs must be provided. From a9458f86baff2353272f74e32177c30a880e92bf Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Fri, 6 Mar 2020 01:26:07 -0500 Subject: [PATCH 08/10] add passing a partial list of dims --- xarray/core/dataarray.py | 2 +- xarray/core/dataset.py | 7 ++++--- xarray/tests/test_dataset.py | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 474ab1d206d..b8c5a4e27b3 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1705,7 +1705,7 @@ def stack( ---------- dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they - replace. Passing a list with an ellipsis (`[...]`) as existing + replace. Passing a list containing an ellipsis (`[...]`) dimensions will stack over all dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 00e3e45fd59..b7edc7e8be3 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -86,6 +86,7 @@ decode_numpy_dict_values, either_dict_or_kwargs, hashable, + infix_dims, is_dict_like, is_scalar, maybe_wrap_array, @@ -3235,8 +3236,8 @@ def reorder_levels( return self._replace(variables, indexes=indexes) def _stack_once(self, dims, new_dim): - if list(dims) == [...]: - dims = self.dims + if ... in dims: + dims = list(infix_dims(dims, self.dims)) variables = {} for name, var in self.variables.items(): if name not in dims: @@ -3279,7 +3280,7 @@ def stack( ---------- dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they - replace. Passing a list with an ellipsis (`[...]`) as existing + replace. Passing a list containing an ellipsis (`[...]`) dimensions will stack over all dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index d789e0c067b..ae10abf44f3 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -2875,6 +2875,10 @@ def test_stack(self): actual = ds.stack(z=(...,)) assert_identical(expected, actual) + # ellipsis with given dim + actual = ds.stack(z=[..., "y"]) + assert_identical(expected, actual) + exp_index = pd.MultiIndex.from_product([["a", "b"], [0, 1]], names=["y", "x"]) expected = Dataset( {"a": ("z", [0, 1, 0, 1]), "b": ("z", [0, 2, 1, 3]), "z": exp_index} From 4555f52372d4880328c16f1f9f424db82b353ac6 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 19 Mar 2020 15:26:08 -0400 Subject: [PATCH 09/10] more wording changes --- doc/reshaping.rst | 4 ++-- doc/whats-new.rst | 2 +- xarray/core/dataarray.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/reshaping.rst b/doc/reshaping.rst index 90a6efe3414..465ca14dfc2 100644 --- a/doc/reshaping.rst +++ b/doc/reshaping.rst @@ -109,11 +109,11 @@ implemented :py:meth:`~xarray.DataArray.stack` and stacked stacked.unstack('z') -As elsewhere in xarray, an ellipsis (`...`) can be used to represent all dimensions: +As elsewhere in xarray, an ellipsis (`...`) can be used to represent all unlisted dimensions: .. ipython:: python - stacked = array.stack(z=[...]) + stacked = array.stack(z=[..., "x"]) stacked These methods are modeled on the :py:class:`pandas.DataFrame` methods of the diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 6517c64ee1c..6082079870f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -46,7 +46,7 @@ New Features By `Julia Signell `_. - An ellipsis (``...``) is now supported in the ``dims`` argument of :py:meth:`Dataset.stack` and :py:meth:`DataArray.stack`, meaning all - dimensions, similar to its meaning in :py:meth:`DataArray.transpose`. + unlisted dimensions, similar to its meaning in :py:meth:`DataArray.transpose`. (:pull:`3826`) By `Maximilian Roos `_ - :py:meth:`Dataset.where` and :py:meth:`DataArray.where` accept a lambda as a diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 0ded38fd81d..686074f5550 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1707,8 +1707,7 @@ def stack( ---------- dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they - replace. Passing a list containing an ellipsis (`[...]`) - dimensions will stack over all dimensions. + replace. An ellipsis (`...`) will be replaced by all unlisted dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. One of dimensions or dimensions_kwargs must be provided. From ab68943aec83b91c77774460de83090db7466a28 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Thu, 19 Mar 2020 17:34:10 -0400 Subject: [PATCH 10/10] improvement from @dcherian --- xarray/core/dataarray.py | 2 ++ xarray/core/dataset.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 86eebf28ea9..324e7ccd290 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1710,6 +1710,8 @@ def stack( dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they replace. An ellipsis (`...`) will be replaced by all unlisted dimensions. + Passing a list containing an ellipsis (`stacked_dim=[...]`) will stack over + all dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. One of dimensions or dimensions_kwargs must be provided. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index be03ba9c4ea..4a369b9aa81 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -3307,8 +3307,9 @@ def stack( ---------- dimensions : Mapping of the form new_name=(dim1, dim2, ...) Names of new dimensions, and the existing dimensions that they - replace. Passing a list containing an ellipsis (`[...]`) - dimensions will stack over all dimensions. + replace. An ellipsis (`...`) will be replaced by all unlisted dimensions. + Passing a list containing an ellipsis (`stacked_dim=[...]`) will stack over + all dimensions. **dimensions_kwargs: The keyword arguments form of ``dimensions``. One of dimensions or dimensions_kwargs must be provided.