From 10030ff27c6453084b4df23a72aba8f1a8140c30 Mon Sep 17 00:00:00 2001 From: matthmey Date: Tue, 10 Dec 2019 15:35:00 +0100 Subject: [PATCH 1/3] bug fixed and added zarr group tests --- xarray/backends/zarr.py | 4 ++-- xarray/tests/test_backends.py | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 6d4ebb02a11..1ebe4e71a1f 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -372,7 +372,7 @@ def store( if len(existing_variables) > 0: # there are variables to append # their encoding must be the same as in the store - ds = open_zarr(self.ds.store, chunks=None) + ds = open_zarr(self.ds.store, group=self.ds.path, chunks=None) variables_with_encoding = {} for vn in existing_variables: variables_with_encoding[vn] = variables[vn].copy(deep=False) @@ -486,7 +486,7 @@ def open_zarr( directory in file system where a Zarr DirectoryStore has been stored. synchronizer : object, optional Array synchronizer provided to zarr - group : str, obtional + group : str, optional Group path. (a.k.a. `path` in zarr terminology.) chunks : int or dict or tuple or {None, 'auto'}, optional Chunk sizes along each dimension, e.g., ``5`` or diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a23527bd49a..9374888f658 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1723,39 +1723,40 @@ def test_hidden_zarr_keys(self): with xr.decode_cf(store): pass - def test_write_persistence_modes(self): + @pytest.mark.parametrize("group", [None, "group1"]) + def test_write_persistence_modes(self,group): original = create_test_data() # overwrite mode - with self.roundtrip(original, save_kwargs={"mode": "w"}) as actual: + with self.roundtrip(original, save_kwargs={"mode": "w","group":group}, open_kwargs={"group":group}) as actual: assert_identical(original, actual) # don't overwrite mode - with self.roundtrip(original, save_kwargs={"mode": "w-"}) as actual: + with self.roundtrip(original, save_kwargs={"mode": "w-","group":group}, open_kwargs={"group":group}) as actual: assert_identical(original, actual) # make sure overwriting works as expected with self.create_zarr_target() as store: self.save(original, store) # should overwrite with no error - self.save(original, store, mode="w") - with self.open(store) as actual: + self.save(original, store, mode="w", group=group) + with self.open(store, group=group) as actual: assert_identical(original, actual) with pytest.raises(ValueError): self.save(original, store, mode="w-") # check append mode for normal write - with self.roundtrip(original, save_kwargs={"mode": "a"}) as actual: + with self.roundtrip(original, save_kwargs={"mode": "a","group":group}, open_kwargs={"group":group}) as actual: assert_identical(original, actual) - ds, ds_to_append, _ = create_append_test_data() - # check append mode for append write + ds, ds_to_append, _ = create_append_test_data() with self.create_zarr_target() as store_target: - ds.to_zarr(store_target, mode="w") - ds_to_append.to_zarr(store_target, append_dim="time") + ds.to_zarr(store_target, mode="w", group=group) + ds_to_append.to_zarr(store_target, append_dim="time", group=group) original = xr.concat([ds, ds_to_append], dim="time") - assert_identical(original, xr.open_zarr(store_target)) + actual = xr.open_zarr(store_target,group=group) + assert_identical(original, actual) def test_compressor_encoding(self): original = create_test_data() From f95b028458c5929c5b43840f26bf2d60c7a2f297 Mon Sep 17 00:00:00 2001 From: matthmey Date: Tue, 10 Dec 2019 15:39:25 +0100 Subject: [PATCH 2/3] black . --- xarray/tests/test_backends.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 9374888f658..2252ce3f3b7 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1724,15 +1724,23 @@ def test_hidden_zarr_keys(self): pass @pytest.mark.parametrize("group", [None, "group1"]) - def test_write_persistence_modes(self,group): + def test_write_persistence_modes(self, group): original = create_test_data() # overwrite mode - with self.roundtrip(original, save_kwargs={"mode": "w","group":group}, open_kwargs={"group":group}) as actual: + with self.roundtrip( + original, + save_kwargs={"mode": "w", "group": group}, + open_kwargs={"group": group}, + ) as actual: assert_identical(original, actual) # don't overwrite mode - with self.roundtrip(original, save_kwargs={"mode": "w-","group":group}, open_kwargs={"group":group}) as actual: + with self.roundtrip( + original, + save_kwargs={"mode": "w-", "group": group}, + open_kwargs={"group": group}, + ) as actual: assert_identical(original, actual) # make sure overwriting works as expected @@ -1746,16 +1754,20 @@ def test_write_persistence_modes(self,group): self.save(original, store, mode="w-") # check append mode for normal write - with self.roundtrip(original, save_kwargs={"mode": "a","group":group}, open_kwargs={"group":group}) as actual: + with self.roundtrip( + original, + save_kwargs={"mode": "a", "group": group}, + open_kwargs={"group": group}, + ) as actual: assert_identical(original, actual) # check append mode for append write - ds, ds_to_append, _ = create_append_test_data() + ds, ds_to_append, _ = create_append_test_data() with self.create_zarr_target() as store_target: ds.to_zarr(store_target, mode="w", group=group) ds_to_append.to_zarr(store_target, append_dim="time", group=group) original = xr.concat([ds, ds_to_append], dim="time") - actual = xr.open_zarr(store_target,group=group) + actual = xr.open_zarr(store_target, group=group) assert_identical(original, actual) def test_compressor_encoding(self): From 2152b42cee4855eaa3baf558401a1d3f981168e3 Mon Sep 17 00:00:00 2001 From: niowniow Date: Fri, 10 Jan 2020 15:30:31 +0100 Subject: [PATCH 3/3] added info to whats-new --- doc/whats-new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 5a9f2497ed6..4d83512d162 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -70,6 +70,8 @@ Bug fixes By `Justus Magin `_. - :py:meth:`Dataset.rename`, :py:meth:`DataArray.rename` now check for conflicts with MultiIndex level names. +- Fix :py:meth:`xarray.core.dataset.Dataset.to_zarr` when using `append_dim` and `group` + simultaneously. (:issue:`3170`). By `Matthias Meyer `_. Documentation ~~~~~~~~~~~~~