diff --git a/conda/meta.yaml b/conda/meta.yaml index e0e75d6..0001bb5 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,5 +1,5 @@ {% set name = "tethysts" %} -{% set version = "4.5.13" %} +{% set version = "4.5.14" %} # {% set sha256 = "ae2cc83fb5a75e8dc3e1b2c2137deea412c8a4c7c9acca52bf4ec59de52a80c9" %} # sha256 is the prefered checksum -- you can get it for a file with: @@ -38,7 +38,7 @@ requirements: - python >=3.8 - pandas<2 - tethys-data-models=0.4.11 - - hdf5tools>=0.1.14 + - hdf5tools>=0.2.3 - s3tethys>=0.0.8 - xarray>=2022.6.0 - pydantic=1.10 diff --git a/setup.py b/setup.py index 25a174f..c43b537 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ name = 'tethysts' main_package = 'tethysts' datasets = 'datasets/time_series' -version = '4.5.13' +version = '4.5.14' descrip = 'tethys time series S3 extraction' # The below code is for readthedocs. To have sphinx/readthedocs interact with @@ -19,7 +19,7 @@ if os.environ.get('READTHEDOCS', False) == 'True': INSTALL_REQUIRES = [] else: - INSTALL_REQUIRES = ['zstandard', 'pandas<2', 'xarray>=2022.6.0', 'scipy', 'orjson', 'requests', 'shapely>=2.0.1', 'tethys-data-models>=0.4.11', 'hdf5tools>=0.1.14', 's3tethys>=0.0.8', 'pydantic==1.10', 'h5netcdf>=1.1.0'] + INSTALL_REQUIRES = ['zstandard', 'pandas<2', 'xarray>=2022.6.0', 'scipy', 'orjson', 'requests', 'shapely>=2.0.1', 'tethys-data-models>=0.4.11', 'hdf5tools>=0.2.3', 's3tethys>=0.0.8', 'pydantic==1.10', 'h5netcdf>=1.1.0'] # Get the long description from the README file with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: diff --git a/tethysts/utils.py b/tethysts/utils.py index 7d4db7f..5f91190 100644 --- a/tethysts/utils.py +++ b/tethysts/utils.py @@ -9,6 +9,7 @@ from datetime import datetime import zstandard as zstd import copy +import pickle import botocore from shapely.geometry import shape, Polygon, Point from shapely.strtree import STRtree @@ -36,6 +37,39 @@ ### Helper functions +def read_pkl_zstd(obj, unpickle=False): + """ + Deserializer from a pickled object compressed with zstandard. + + Parameters + ---------- + obj : bytes or str + Either a bytes object that has been pickled and compressed or a str path to the file object. + unpickle : bool + Should the bytes object be unpickled or left as bytes? + + Returns + ------- + Python object + """ + if isinstance(obj, str): + with open(obj, 'rb') as p: + dctx = zstd.ZstdDecompressor() + with dctx.stream_reader(p) as reader: + obj1 = reader.read() + + elif isinstance(obj, bytes): + dctx = zstd.ZstdDecompressor() + obj1 = dctx.decompress(obj) + else: + raise TypeError('obj must either be a str path or a bytes object') + + if unpickle: + obj1 = pickle.loads(obj1) + + return obj1 + + def update_nested(in_dict, ds_id, version_date, value): """