From 035678c4dfde180f51d0143bed0613b2e4dfa5c5 Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Mon, 26 Sep 2022 12:41:03 -0400 Subject: [PATCH 01/12] Switched from netCDF4 to h5netcdf --- arviz/data/inference_data.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arviz/data/inference_data.py b/arviz/data/inference_data.py index 346289b8f7..8710678ed7 100644 --- a/arviz/data/inference_data.py +++ b/arviz/data/inference_data.py @@ -24,7 +24,7 @@ overload, ) -import netCDF4 as nc +import h5netcdf.legacyapi as nc import numpy as np import xarray as xr from packaging import version diff --git a/requirements.txt b/requirements.txt index 65940c462c..92ac00506c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ scipy>=1.8.0 packaging pandas>=1.4.0 xarray>=0.21.0 -netcdf4 +h5netcdf==1.0.2 typing_extensions>=4.1.0 xarray-einstats>=0.3 From b1a2b52243d27890ca1ae591f63b3df5d7cb9a5c Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Mon, 26 Sep 2022 12:47:32 -0400 Subject: [PATCH 02/12] Removed unnecessary kwarg --- arviz/data/inference_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arviz/data/inference_data.py b/arviz/data/inference_data.py index 8710678ed7..7afcf226d4 100644 --- a/arviz/data/inference_data.py +++ b/arviz/data/inference_data.py @@ -24,7 +24,7 @@ overload, ) -import h5netcdf.legacyapi as nc +import h5netcdf.legacyapi as nc import numpy as np import xarray as xr from packaging import version @@ -445,7 +445,7 @@ def to_netcdf( data.close() mode = "a" elif not self._attrs: # creates a netcdf file for an empty InferenceData object. - empty_netcdf_file = nc.Dataset(filename, mode="w", format="NETCDF4") + empty_netcdf_file = nc.Dataset(filename, mode="w") empty_netcdf_file.close() return filename From 50b1b79d9c0b7adfe8c65e7c1e0649c51227460d Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Mon, 26 Sep 2022 12:50:36 -0400 Subject: [PATCH 03/12] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29cbef40c6..4d6c380f00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ * Update tests and docs for updated example data ([2137](https://github.com/arviz-devs/arviz/pull/2137)) * Copy coords before modifying in ppcplot ([2160](https://github.com/arviz-devs/arviz/pull/2160)) +* Changed dependency on netCDF4 to h5netcdf ### Deprecation * Removed `fill_last`, `contour` and `plot_kwargs` arguments from `plot_pair` function ([2085](https://github.com/arviz-devs/arviz/pull/2085)) From 0b327aa96599a078d8ced05bd414fd7506c24b1b Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Tue, 27 Sep 2022 09:57:14 -0400 Subject: [PATCH 04/12] passing the engine option to open_dataset --- arviz/data/inference_data.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/arviz/data/inference_data.py b/arviz/data/inference_data.py index 7afcf226d4..f43d1cffb7 100644 --- a/arviz/data/inference_data.py +++ b/arviz/data/inference_data.py @@ -23,8 +23,13 @@ Union, overload, ) +try: + import h5netcdf.legacyapi as nc + HAS_H5NETCDF = True +except ImportError: + import netCDF4 as nc + HAS_H5NETCDF = False -import h5netcdf.legacyapi as nc import numpy as np import xarray as xr from packaging import version @@ -371,12 +376,16 @@ def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": for group in data_groups: group_kws = {} + + if group_kwargs is not None and regex is False: group_kws = group_kwargs.get(group, {}) if group_kwargs is not None and regex is True: for key, kws in group_kwargs.items(): if re.search(key, group): group_kws = kws + if HAS_H5NETCDF: + group_kws['engine'] = 'h5netcdf' with xr.open_dataset(filename, group=group, **group_kws) as data: if rcParams["data.load"] == "eager": groups[group] = data.load() From 312b93d36c5cfbb400fc64fd7300445abc309ac4 Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Tue, 27 Sep 2022 10:18:09 -0400 Subject: [PATCH 05/12] added engine kwargs to "to_netcdf" --- arviz/data/inference_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arviz/data/inference_data.py b/arviz/data/inference_data.py index f43d1cffb7..8e3fe5d38e 100644 --- a/arviz/data/inference_data.py +++ b/arviz/data/inference_data.py @@ -450,6 +450,8 @@ def to_netcdf( for var_name, values in data.variables.items() if _compressible_dtype(values.dtype) } + if HAS_H5NETCDF: + kwargs['engine'] = 'h5netcdf' data.to_netcdf(filename, mode=mode, group=group, **kwargs) data.close() mode = "a" From ad18daa705a3709a69d86569ac46a8d0c784742c Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Tue, 27 Sep 2022 10:28:20 -0400 Subject: [PATCH 06/12] addressed an edge case where group_kws is not initialized --- arviz/data/inference_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arviz/data/inference_data.py b/arviz/data/inference_data.py index 8e3fe5d38e..88efb6b0d8 100644 --- a/arviz/data/inference_data.py +++ b/arviz/data/inference_data.py @@ -377,7 +377,7 @@ def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": for group in data_groups: group_kws = {} - + group_kws = {} if group_kwargs is not None and regex is False: group_kws = group_kwargs.get(group, {}) if group_kwargs is not None and regex is True: From 595b559262a5e3c27f7d4912d29a47353c8eb351 Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Wed, 26 Oct 2022 14:57:15 -0400 Subject: [PATCH 07/12] removed legacy api --- arviz/data/inference_data.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/arviz/data/inference_data.py b/arviz/data/inference_data.py index 88efb6b0d8..b8fd264530 100644 --- a/arviz/data/inference_data.py +++ b/arviz/data/inference_data.py @@ -23,12 +23,7 @@ Union, overload, ) -try: - import h5netcdf.legacyapi as nc - HAS_H5NETCDF = True -except ImportError: - import netCDF4 as nc - HAS_H5NETCDF = False +import h5netcdf import numpy as np import xarray as xr @@ -371,21 +366,20 @@ def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": attrs = {} try: - with nc.Dataset(filename, mode="r") as data: + with h5netcdf.File(filename, mode="r") as data: data_groups = list(data.groups) for group in data_groups: group_kws = {} - group_kws = {} + group_kws = {'engine':'h5netcdf'} if group_kwargs is not None and regex is False: group_kws = group_kwargs.get(group, {}) if group_kwargs is not None and regex is True: for key, kws in group_kwargs.items(): if re.search(key, group): group_kws = kws - if HAS_H5NETCDF: - group_kws['engine'] = 'h5netcdf' + with xr.open_dataset(filename, group=group, **group_kws) as data: if rcParams["data.load"] == "eager": groups[group] = data.load() @@ -443,20 +437,18 @@ def to_netcdf( for group in groups: data = getattr(self, group) - kwargs = {} + kwargs = {'engine':'h5netcdf'} if compress: kwargs["encoding"] = { var_name: {"zlib": True} for var_name, values in data.variables.items() if _compressible_dtype(values.dtype) } - if HAS_H5NETCDF: - kwargs['engine'] = 'h5netcdf' data.to_netcdf(filename, mode=mode, group=group, **kwargs) data.close() mode = "a" elif not self._attrs: # creates a netcdf file for an empty InferenceData object. - empty_netcdf_file = nc.Dataset(filename, mode="w") + empty_netcdf_file = h5netcdf.File(filename, mode="w") empty_netcdf_file.close() return filename From 1b8580635713056078fff8890668253ccf07fb41 Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Sat, 15 Oct 2022 17:30:26 -0400 Subject: [PATCH 08/12] Update requirements.txt Co-authored-by: Ari Hartikainen --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 92ac00506c..bc6e88ec94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ scipy>=1.8.0 packaging pandas>=1.4.0 xarray>=0.21.0 -h5netcdf==1.0.2 +h5netcdf>=1.0.2 typing_extensions>=4.1.0 xarray-einstats>=0.3 From 88f54f6cceacdba5c1e33ee65add44495e0650d2 Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Wed, 26 Oct 2022 14:25:46 -0400 Subject: [PATCH 09/12] Update CHANGELOG.md Co-authored-by: Ari Hartikainen --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d6c380f00..84fd20cd26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ * Copy coords before modifying in ppcplot ([2160](https://github.com/arviz-devs/arviz/pull/2160)) * Changed dependency on netCDF4 to h5netcdf +* Changed dependency on netCDF4 to h5netcdf ([2122](https://github.com/arviz-devs/arviz/pull/2122)) + ### Deprecation * Removed `fill_last`, `contour` and `plot_kwargs` arguments from `plot_pair` function ([2085](https://github.com/arviz-devs/arviz/pull/2085)) From ed7b6c517f9ab90f30ea012ca72634dde2ea0340 Mon Sep 17 00:00:00 2001 From: Varchas Gopalaswamy Date: Wed, 26 Oct 2022 14:59:47 -0400 Subject: [PATCH 10/12] put setdefault back --- arviz/data/inference_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arviz/data/inference_data.py b/arviz/data/inference_data.py index b8fd264530..2fd05502cc 100644 --- a/arviz/data/inference_data.py +++ b/arviz/data/inference_data.py @@ -372,14 +372,14 @@ def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": for group in data_groups: group_kws = {} - group_kws = {'engine':'h5netcdf'} + group_kws = {} if group_kwargs is not None and regex is False: group_kws = group_kwargs.get(group, {}) if group_kwargs is not None and regex is True: for key, kws in group_kwargs.items(): if re.search(key, group): group_kws = kws - + group_kws.setdefault("engine", "h5netcdf") with xr.open_dataset(filename, group=group, **group_kws) as data: if rcParams["data.load"] == "eager": groups[group] = data.load() From 7d8625077b3937e52c174307e08f6e65d4e2778f Mon Sep 17 00:00:00 2001 From: "Oriol (ZBook)" Date: Thu, 22 Dec 2022 04:45:10 +0100 Subject: [PATCH 11/12] allow both h5netcdf and netcdf4 to be used --- CHANGELOG.md | 4 +-- arviz/data/inference_data.py | 49 ++++++++++++++++++++++------- arviz/data/io_netcdf.py | 14 ++++++--- arviz/tests/base_tests/test_data.py | 13 +++++++- requirements-optional.txt | 1 + requirements-test.txt | 2 +- 6 files changed, 63 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84fd20cd26..99ced2db12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### New features - Adds Savage-Dickey density ratio plot for Bayes factor approximation. ([2037](https://github.com/arviz-devs/arviz/pull/2037), [2152](https://github.com/arviz-devs/arviz/pull/2152)) - Add `CmdStanPySamplingWrapper` and `PyMCSamplingWrapper` classes ([2158](https://github.com/arviz-devs/arviz/pull/2158)) +- Changed dependency on netcdf4-python to h5netcdf ([2122](https://github.com/arviz-devs/arviz/pull/2122)) ### Maintenance and fixes - Fix `reloo` outdated usage of `ELPDData` ([2158](https://github.com/arviz-devs/arviz/pull/2158)) @@ -55,9 +56,6 @@ * Update tests and docs for updated example data ([2137](https://github.com/arviz-devs/arviz/pull/2137)) * Copy coords before modifying in ppcplot ([2160](https://github.com/arviz-devs/arviz/pull/2160)) -* Changed dependency on netCDF4 to h5netcdf -* Changed dependency on netCDF4 to h5netcdf ([2122](https://github.com/arviz-devs/arviz/pull/2122)) - ### Deprecation * Removed `fill_last`, `contour` and `plot_kwargs` arguments from `plot_pair` function ([2085](https://github.com/arviz-devs/arviz/pull/2085)) diff --git a/arviz/data/inference_data.py b/arviz/data/inference_data.py index 2fd05502cc..8a273b731f 100644 --- a/arviz/data/inference_data.py +++ b/arviz/data/inference_data.py @@ -23,7 +23,6 @@ Union, overload, ) -import h5netcdf import numpy as np import xarray as xr @@ -337,7 +336,9 @@ def items(self) -> "InferenceData.InferenceDataItemsView": return InferenceData.InferenceDataItemsView(self) @staticmethod - def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": + def from_netcdf( + filename, *, engine="h5netcdf", group_kwargs=None, regex=False + ) -> "InferenceData": """Initialize object from a netcdf file. Expects that the file will have groups, each of which can be loaded by xarray. @@ -349,6 +350,8 @@ def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": ---------- filename : str location of netcdf file + engine : {"h5netcdf", "netcdf4"}, default "h5netcdf" + Library used to read the netcdf file. group_kwargs : dict of {str: dict}, optional Keyword arguments to be passed into each call of :func:`xarray.open_dataset`. The keys of the higher level should be group names or regex matching group @@ -360,13 +363,24 @@ def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": Returns ------- - InferenceData object + InferenceData """ groups = {} attrs = {} + if engine == "h5netcdf": + import h5netcdf + elif engine == "netcdf4": + import netCDF4 as nc + else: + raise ValueError( + f"Invalid value for engine: {engine}. Valid options are: h5netcdf or netcdf4" + ) + try: - with h5netcdf.File(filename, mode="r") as data: + with h5netcdf.File(filename, mode="r") if engine == "h5netcdf" else nc.Dataset( + filename, mode="r" + ) as data: data_groups = list(data.groups) for group in data_groups: @@ -379,14 +393,14 @@ def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": for key, kws in group_kwargs.items(): if re.search(key, group): group_kws = kws - group_kws.setdefault("engine", "h5netcdf") + group_kws.setdefault("engine", engine) with xr.open_dataset(filename, group=group, **group_kws) as data: if rcParams["data.load"] == "eager": groups[group] = data.load() else: groups[group] = data - with xr.open_dataset(filename, mode="r") as data: + with xr.open_dataset(filename, engine=engine) as data: attrs.update(data.load().attrs) return InferenceData(attrs=attrs, **groups) @@ -405,9 +419,13 @@ def from_netcdf(filename, group_kwargs=None, regex=False) -> "InferenceData": raise err def to_netcdf( - self, filename: str, compress: bool = True, groups: Optional[List[str]] = None + self, + filename: str, + compress: bool = True, + groups: Optional[List[str]] = None, + engine: str = "h5netcdf", ) -> str: - """Write InferenceData to file using netcdf4. + """Write InferenceData to netcdf4 file. Parameters ---------- @@ -418,6 +436,8 @@ def to_netcdf( saving and loading somewhat slower (default: True). groups : list, optional Write only these groups to netcdf file. + engine : {"h5netcdf", "netcdf4"}, default "h5netcdf" + Library used to read the netcdf file. Returns ------- @@ -426,7 +446,7 @@ def to_netcdf( """ mode = "w" # overwrite first, then append if self._attrs: - xr.Dataset(attrs=self._attrs).to_netcdf(filename, mode=mode) + xr.Dataset(attrs=self._attrs).to_netcdf(filename, mode=mode, engine=engine) mode = "a" if self._groups_all: # check's whether a group is present or not. @@ -437,7 +457,7 @@ def to_netcdf( for group in groups: data = getattr(self, group) - kwargs = {'engine':'h5netcdf'} + kwargs = {"engine": engine} if compress: kwargs["encoding"] = { var_name: {"zlib": True} @@ -448,7 +468,14 @@ def to_netcdf( data.close() mode = "a" elif not self._attrs: # creates a netcdf file for an empty InferenceData object. - empty_netcdf_file = h5netcdf.File(filename, mode="w") + if engine == "h5netcdf": + import h5netcdf + + empty_netcdf_file = h5netcdf.File(filename, mode="w") + elif engine == "netcdf4": + import netCDF4 as nc + + empty_netcdf_file = nc.Dataset(filename, mode="w", format="NETCDF4") empty_netcdf_file.close() return filename diff --git a/arviz/data/io_netcdf.py b/arviz/data/io_netcdf.py index 93574a6cdd..bd20e20921 100644 --- a/arviz/data/io_netcdf.py +++ b/arviz/data/io_netcdf.py @@ -4,13 +4,15 @@ from .inference_data import InferenceData -def from_netcdf(filename, group_kwargs=None, regex=False): +def from_netcdf(filename, *, engine="h5netcdf", group_kwargs=None, regex=False): """Load netcdf file back into an arviz.InferenceData. Parameters ---------- filename : str name or path of the file to load trace + engine : {"h5netcdf", "netcdf4"}, default "h5netcdf" + Library used to read the netcdf file. group_kwargs : dict of {str: dict} Keyword arguments to be passed into each call of :func:`xarray.open_dataset`. The keys of the higher level should be group names or regex matching group @@ -31,10 +33,12 @@ def from_netcdf(filename, group_kwargs=None, regex=False): """ if group_kwargs is None: group_kwargs = {} - return InferenceData.from_netcdf(filename, group_kwargs, regex) + return InferenceData.from_netcdf( + filename, engine=engine, group_kwargs=group_kwargs, regex=regex + ) -def to_netcdf(data, filename, *, group="posterior", coords=None, dims=None): +def to_netcdf(data, filename, *, group="posterior", engine="h5netcdf", coords=None, dims=None): """Save dataset as a netcdf file. WARNING: Only idempotent in case `data` is InferenceData @@ -47,6 +51,8 @@ def to_netcdf(data, filename, *, group="posterior", coords=None, dims=None): name or path of the file to load trace group : str (optional) In case `data` is not InferenceData, this is the group it will be saved to + engine : {"h5netcdf", "netcdf4"}, default "h5netcdf" + Library used to read the netcdf file. coords : dict (optional) See `convert_to_inference_data` dims : dict (optional) @@ -58,5 +64,5 @@ def to_netcdf(data, filename, *, group="posterior", coords=None, dims=None): filename saved to """ inference_data = convert_to_inference_data(data, group=group, coords=coords, dims=dims) - file_name = inference_data.to_netcdf(filename) + file_name = inference_data.to_netcdf(filename, engine=engine) return file_name diff --git a/arviz/tests/base_tests/test_data.py b/arviz/tests/base_tests/test_data.py index cb637ea3ff..3e4fbce322 100644 --- a/arviz/tests/base_tests/test_data.py +++ b/arviz/tests/base_tests/test_data.py @@ -1296,11 +1296,22 @@ def test_io_function(self, data, eight_schools_params): @pytest.mark.parametrize("groups_arg", [False, True]) @pytest.mark.parametrize("compress", [True, False]) - def test_io_method(self, data, eight_schools_params, groups_arg, compress): + @pytest.mark.parametrize("engine", ["h5netcdf", "netcdf4"]) + def test_io_method(self, data, eight_schools_params, groups_arg, compress, engine): # create InferenceData and check it has been properly created inference_data = self.get_inference_data( # pylint: disable=W0612 data, eight_schools_params ) + if engine == "h5netcdf": + try: + import h5netcdf # pylint: disable=unused-import + except ImportError: + pytest.skip("h5netcdf not installed") + elif engine == "netcdf4": + try: + import netCDF4 # pylint: disable=unused-import + except ImportError: + pytest.skip("netcdf4 not installed") test_dict = { "posterior": ["eta", "theta", "mu", "tau"], "posterior_predictive": ["eta", "theta", "mu", "tau"], diff --git a/requirements-optional.txt b/requirements-optional.txt index d870218fe5..b0b470a41e 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,4 +1,5 @@ numba +netcdf4 bokeh>=1.4.0,<3.0 contourpy ujson diff --git a/requirements-test.txt b/requirements-test.txt index 25a69e0498..6a22ea6482 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,4 @@ pytest-cov cloudpickle -r requirements-optional.txt --r requirements-external.txt \ No newline at end of file +-r requirements-external.txt From f78256171aaee87d964145270f0cac9c041decd5 Mon Sep 17 00:00:00 2001 From: "Oriol (ZBook)" Date: Thu, 22 Dec 2022 04:55:22 +0100 Subject: [PATCH 12/12] fix dataset loading --- arviz/data/datasets.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/arviz/data/datasets.py b/arviz/data/datasets.py index fc7c242824..7ee857dfcf 100644 --- a/arviz/data/datasets.py +++ b/arviz/data/datasets.py @@ -85,7 +85,7 @@ def _sha256(path): return sha256hash.hexdigest() -def load_arviz_data(dataset=None, data_home=None, regex=False, **kwargs): +def load_arviz_data(dataset=None, data_home=None, **kwargs): """Load a local or remote pre-made dataset. Run with no parameters to get a list of all available models. @@ -100,17 +100,10 @@ def load_arviz_data(dataset=None, data_home=None, regex=False, **kwargs): ---------- dataset : str Name of dataset to load. - data_home : str, optional Where to save remote datasets - - regex : bool, optional - Specifies regex support for chunking information in - :func:`arviz.from_netcdf`. This feature is currently experimental. - - **kwargs : dict of {str: dict}, optional - Keyword arguments to be passed to :func:`arviz.from_netcdf`. - This feature is currently experimental. + **kwargs : dict, optional + Keyword arguments passed to :func:`arviz.from_netcdf`. Returns ------- @@ -119,7 +112,7 @@ def load_arviz_data(dataset=None, data_home=None, regex=False, **kwargs): """ if dataset in LOCAL_DATASETS: resource = LOCAL_DATASETS[dataset] - return from_netcdf(resource.filename) + return from_netcdf(resource.filename, **kwargs) elif dataset in REMOTE_DATASETS: remote = REMOTE_DATASETS[dataset] @@ -140,7 +133,7 @@ def load_arviz_data(dataset=None, data_home=None, regex=False, **kwargs): "({remote.checksum}), file may be corrupted. " "Run `arviz.clear_data_home()` and try again, or please open an issue." ) - return from_netcdf(file_path, kwargs, regex) + return from_netcdf(file_path, **kwargs) else: if dataset is None: return dict(itertools.chain(LOCAL_DATASETS.items(), REMOTE_DATASETS.items()))