From b2762308c29443d3019c0b9e574994ad6dc64f6c Mon Sep 17 00:00:00 2001 From: Hauke Schulz <43613877+observingClouds@users.noreply.github.com> Date: Sat, 4 Feb 2023 13:36:15 -0800 Subject: [PATCH 1/4] Implement dropping of encoding added by xarray in 'to_zarr' --- xarray/backends/zarr.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index e30e7e9f4d8..ae400c2b340 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -230,6 +230,7 @@ def extract_zarr_variable_encoding( """ encoding = variable.encoding.copy() + safe_to_drop = {"source", "original_shape"} valid_encodings = { "chunks", "compressor", @@ -238,6 +239,10 @@ def extract_zarr_variable_encoding( "write_empty_chunks", } + for k in safe_to_drop: + if k in encoding: + del encoding[k] + if raise_on_invalid: invalid = [k for k in encoding if k not in valid_encodings] if invalid: From d149ef1a226389dd38dc81539c5039218c18e479 Mon Sep 17 00:00:00 2001 From: Hauke Schulz <43613877+observingClouds@users.noreply.github.com> Date: Sat, 4 Feb 2023 13:47:00 -0800 Subject: [PATCH 2/4] add test xarray added encoding is dropped in 'to_zarr' --- xarray/tests/test_backends.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index b49500bd00c..c0dace53976 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -2039,6 +2039,12 @@ def test_chunk_encoding_with_dask(self) -> None: # don't actually check equality because the data could be corrupted pass + def test_drop_encoding(self): + ds = open_example_dataset("example_1.nc") + encodings = {v: {**ds[v].encoding} for v in ds.data_vars} + with self.create_zarr_target() as store: + ds.to_zarr(store, encoding=encodings) + def test_hidden_zarr_keys(self) -> None: expected = create_test_data() with self.create_store() as store: From ee64fd0322a67fa4509ab27b000d467253f07bbc Mon Sep 17 00:00:00 2001 From: Hauke Schulz <43613877+observingClouds@users.noreply.github.com> Date: Sat, 4 Feb 2023 14:02:49 -0800 Subject: [PATCH 3/4] add changes to whats new --- doc/whats-new.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 26bd72b0727..aa218678b9d 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -55,6 +55,9 @@ Bug fixes By `Kai Mühlbauer `_ and `Scott Chamberlin `_. - Handle ``keep_attrs`` option in binary operators of :py:meth:`Dataset` (:issue:`7390`, :pull:`7391`). By `Aron Gergely `_. +- :py:func:`xarray.Dataset.to_zarr` drops now variable encodings that have been added by xarray during reading + a dataset. (:issue:`7129`, :pull:`7500`). + By `Hauke Schulz `_. Documentation ~~~~~~~~~~~~~ From dcd13839030bc19ebe19bc4f964ada8ddefc2e1b Mon Sep 17 00:00:00 2001 From: Hauke Schulz <43613877+observingClouds@users.noreply.github.com> Date: Mon, 6 Feb 2023 17:12:26 -0800 Subject: [PATCH 4/4] Fix typo Co-authored-by: Joe Hamman --- doc/whats-new.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index aa218678b9d..458765e9d70 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -55,7 +55,7 @@ Bug fixes By `Kai Mühlbauer `_ and `Scott Chamberlin `_. - Handle ``keep_attrs`` option in binary operators of :py:meth:`Dataset` (:issue:`7390`, :pull:`7391`). By `Aron Gergely `_. -- :py:func:`xarray.Dataset.to_zarr` drops now variable encodings that have been added by xarray during reading +- :py:func:`xarray.Dataset.to_zarr` now drops variable encodings that have been added by xarray during reading a dataset. (:issue:`7129`, :pull:`7500`). By `Hauke Schulz `_.