Merge branch 'main' into doc/v3-technical-design

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,4 +69,4 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/minimal.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Setup Miniconda
-      uses: conda-incubator/setup-miniconda@v2.3.0
+      uses: conda-incubator/setup-miniconda@v3.0.1
       with:
           channels: conda-forge
           environment-file: environment.yml

.github/workflows/python-package.yml

@@ -42,7 +42,7 @@ jobs:
       with:
         fetch-depth: 0
     - name: Setup Miniconda
-      uses: conda-incubator/setup-miniconda@v2.3.0
+      uses: conda-incubator/setup-miniconda@v3.0.1
       with:
           channels: conda-forge
           python-version: ${{ matrix.python-version }}

.github/workflows/releases.yml

@@ -16,7 +16,7 @@ jobs:
           submodules: true
           fetch-depth: 0
 
-      - uses: actions/setup-python@v4.7.1
+      - uses: actions/setup-python@v5.0.0
         name: Install Python
         with:
           python-version: '3.8'
@@ -64,7 +64,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/[email protected].10
+      - uses: pypa/[email protected].11
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.github/workflows/windows-testing.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v2.3.0
+      - uses: conda-incubator/setup-miniconda@v3.0.1
         with:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}

.pre-commit-config.yaml

@@ -6,28 +6,25 @@ default_stages: [commit, push]
 default_language_version:
   python: python3
 repos:
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
+  - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.224'
+    rev: 'v0.1.9'
     hooks:
     - id: ruff
-      # Respect `exclude` and `extend-exclude` settings.
-      args: ["--force-exclude"]
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.12.1
     hooks:
     - id: black
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
       - id: codespell
-        args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.3.0
+    rev: v1.8.0
     hooks:
       - id: mypy
         files: zarr

bench/compress_normal.py

@@ -8,7 +8,6 @@
 from zarr import blosc
 
 if __name__ == "__main__":
-
     sys.path.insert(0, "..")
 
     # setup

docs/release.rst

@@ -18,6 +18,12 @@ Release notes
 Unreleased
 ----------
 
+Enhancements
+~~~~~~~~~~~~
+
+* Added type hints to ``zarr.creation.create()``.
+  By :user:`David Stansby <dstansby>` :issue:`1536`.
+
 Docs
 ~~~~
 
@@ -43,6 +49,9 @@ Docs
 Maintenance
 ~~~~~~~~~~~
 
+* Cache result of ``FSStore._fsspec_installed()``.
+  By :user:`Janick Martinez Esturo <ph03>` :issue:`1581`.
+
 * Extend copyright notice to 2023.
   By :user:`Jack Kelly <JackKelly>` :issue:`1528`.
 

pyproject.toml

@@ -65,6 +65,7 @@ Homepage = "https://github.com/zarr-developers/zarr-python"
 exclude_lines = [
     "pragma: no cover",
     "pragma: ${PY_MAJOR_VERSION} no cover",
+    '.*\.\.\.' # Ignore "..." lines
 ]
 
 [tool.coverage.run]
@@ -119,9 +120,10 @@ exclude = '''
 '''
 
 [tool.mypy]
-python_version = "3.8"
 ignore_missing_imports = true
-follow_imports = "silent"
+warn_unused_configs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
 
 [tool.pytest.ini_options]
 doctest_optionflags = [
@@ -137,3 +139,8 @@ filterwarnings = [
     "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning",
     "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning",
 ]
+
+
+[tool.codespell]
+ignore-words-list = "ba,ihs,kake,nd,noe,nwo,te,fo,zar"
+skip = 'fixture,.git'

requirements_dev_minimal.txt

@@ -1,8 +1,8 @@
 # library requirements
 asciitree==0.3.3
 fasteners==0.19
-numcodecs==0.11.0
+numcodecs==0.12.1
 msgpack-python==0.5.6
 setuptools-scm==8.0.4
 # test requirements
-pytest==7.4.3
+pytest==7.4.4

requirements_dev_optional.txt

@@ -11,13 +11,13 @@ azure-storage-blob==12.16.0 # pyup: ignore
 redis==5.0.1
 types-redis
 types-setuptools
-pymongo==4.5.0
+pymongo==4.6.1
 # optional test requirements
 coverage
 pytest-cov==4.1.0
-pytest-doctestplus==1.0.0
+pytest-doctestplus==1.1.0
 pytest-timeout==2.2.0
 h5py==3.10.0
-fsspec==2023.10.0
-s3fs==2023.10.0
+fsspec==2023.12.2
+s3fs==2023.12.2
 moto[server]>=4.0.8

zarr/_storage/absstore.py

@@ -87,7 +87,7 @@ def __init__(
                 "https://{}.blob.core.windows.net/".format(account_name),
                 container,
                 credential=account_key,
-                **blob_service_kwargs
+                **blob_service_kwargs,
             )
 
         self.client = client
@@ -240,7 +240,6 @@ def __setitem__(self, key, value):
         super().__setitem__(key, value)
 
     def rmdir(self, path=None):
-
         if not path:
             # Currently allowing clear to delete everything as in v2
 

zarr/_storage/store.py

@@ -9,6 +9,7 @@
 from zarr.meta import Metadata2, Metadata3
 from zarr.util import normalize_storage_path
 from zarr.context import Context
+from zarr.types import ZARR_VERSION
 
 # v2 store keys
 array_meta_key = ".zarray"
@@ -19,7 +20,7 @@
 meta_root = "meta/root/"
 data_root = "data/root/"
 
-DEFAULT_ZARR_VERSION = 2
+DEFAULT_ZARR_VERSION: ZARR_VERSION = 2
 
 v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"]
 
@@ -221,9 +222,8 @@ def _validate_key(self, key: str):
             )
 
         if (
-            not key.startswith("data/")
-            and (not key.startswith("meta/"))
-            and (not key == "zarr.json")
+            not key.startswith(("data/", "meta/"))
+            and key != "zarr.json"
             # TODO: Possibly allow key == ".zmetadata" too if we write a
             #       consolidated metadata spec corresponding to this?
         ):
@@ -629,7 +629,6 @@ def _rmdir_from_keys(store: StoreLike, path: Optional[str] = None) -> None:
 
 
 def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None:
-
     meta_dir = meta_root + path
     meta_dir = meta_dir.rstrip("/")
     _rmdir_from_keys(store, meta_dir)
@@ -643,10 +642,10 @@ def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None:
     sfx = _get_metadata_suffix(store)
     array_meta_file = meta_dir + ".array" + sfx
     if array_meta_file in store:
-        store.erase(array_meta_file)  # type: ignore
+        store.erase(array_meta_file)
     group_meta_file = meta_dir + ".group" + sfx
     if group_meta_file in store:
-        store.erase(group_meta_file)  # type: ignore
+        store.erase(group_meta_file)
 
 
 def _listdir_from_keys(store: BaseStore, path: Optional[str] = None) -> List[str]:

zarr/_storage/v3.py

@@ -118,7 +118,6 @@ def _get_files_and_dirs_from_path(store, path):
 
 
 class FSStoreV3(FSStore, StoreV3):
-
     # FSStoreV3 doesn't use this (FSStore uses it within _normalize_key)
     _META_KEYS = ()
 

zarr/_storage/v3_storage_transformers.py

@@ -351,7 +351,7 @@ def erase_prefix(self, prefix):
 
     def rmdir(self, path=None):
         path = normalize_storage_path(path)
-        _rmdir_from_keys_v3(self, path)  # type: ignore
+        _rmdir_from_keys_v3(self, path)
 
     def __contains__(self, key):
         if self._is_data_key(key):

zarr/attrs.py

@@ -26,7 +26,6 @@ class Attributes(MutableMapping):
     """
 
     def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None):
-
         self._version = getattr(store, "_store_version", 2)
         _Store = Store if self._version == 2 else StoreV3
         self.store = _Store._ensure_store(store)
@@ -73,7 +72,6 @@ def __getitem__(self, item):
         return self.asdict()[item]
 
     def _write_op(self, f, *args, **kwargs):
-
         # guard condition
         if self.read_only:
             raise PermissionError("attributes are read-only")
@@ -89,7 +87,6 @@ def __setitem__(self, item, value):
         self._write_op(self._setitem_nosync, item, value)
 
     def _setitem_nosync(self, item, value):
-
         # load existing data
         d = self._get_nosync()
 
@@ -106,7 +103,6 @@ def __delitem__(self, item):
         self._write_op(self._delitem_nosync, item)
 
     def _delitem_nosync(self, key):
-
         # load existing data
         d = self._get_nosync()
 
@@ -128,7 +124,6 @@ def put(self, d):
             self._write_op(self._put_nosync, dict(attributes=d))
 
     def _put_nosync(self, d):
-
         d_to_check = d if self._version == 2 else d["attributes"]
         if not all(isinstance(item, str) for item in d_to_check):
             # TODO: Raise an error for non-string keys
@@ -178,7 +173,6 @@ def update(self, *args, **kwargs):
         self._write_op(self._update_nosync, *args, **kwargs)
 
     def _update_nosync(self, *args, **kwargs):
-
         # load existing data
         d = self._get_nosync()