From daf0d3c309250f59df9d762d0c115762df93d659 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Thu, 2 Nov 2023 15:18:09 +0100 Subject: [PATCH 01/13] Add Python 3.12 to CI --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee00602..971e728 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,12 +6,12 @@ jobs: name: Unit tests - ${{ matrix.PYTHON_VERSION }} runs-on: ubuntu-latest permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" strategy: fail-fast: false matrix: - PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11"] + PYTHON_VERSION: ["3.8", "3.9", "3.10", "3.11", "3.12"] services: postgres: image: postgres:11 @@ -45,12 +45,12 @@ jobs: continue-on-error: true - id: google_auth if: steps.check-id-token.outcome == 'success' - name: 'Authenticate to Google Cloud' + name: "Authenticate to Google Cloud" uses: google-github-actions/auth@v1 with: - workload_identity_provider: 'projects/498651197656/locations/global/workloadIdentityPools/qc-minimalkv-gh-actions-pool/providers/github-actions-provider' - service_account: 'sa-github-actions@qc-minimalkv.iam.gserviceaccount.com' - token_format: 'access_token' + workload_identity_provider: "projects/498651197656/locations/global/workloadIdentityPools/qc-minimalkv-gh-actions-pool/providers/github-actions-provider" + service_account: "sa-github-actions@qc-minimalkv.iam.gserviceaccount.com" + token_format: "access_token" - name: Run the unittests shell: bash -x -l {0} run: | From 2b10cbbd3d580bd44e4e4524a8c8fe97e687d54a Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Thu, 2 Nov 2023 16:40:07 +0100 Subject: [PATCH 02/13] Port project completely to pyproject.toml --- pyproject.toml | 36 +++++++++++++++++++++++++++++++++++- setup.cfg | 2 -- setup.py | 35 ----------------------------------- 3 files changed, 35 insertions(+), 38 deletions(-) delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index 1826ffc..0388299 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,39 @@ [build-system] -requires = ['setuptools', 'setuptools-scm', 'wheel'] +requires = ["setuptools", "setuptools_scm", "wheel"] + +[tool.setuptools_scm] +version_scheme = "post-release" + +[project] +name = "minimalkv" +dynamic = ["version"] +license = { file = "LICENSE" } +description = "A key-value storage for binary data, support many backends." +readme = "README.rst" +authors = [{name = "Data Engineering Collective", email = "minimalkv@uwekorn.com"}] +classifiers = [ + "License :: OSI Approved :: BSD License", + "License :: OSI Approved :: BSD-3 Clause", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Development Status :: 5 - Production/Stable", + "Operating System :: OS Independent", +] +dependencies = ["uritools"] +requires-python = ">=3.8" + +[project.urls] +repository = "https://github.com/data-engineering-collective/minimalkv" + +[tool.setuptools.packages.find] +include = ["minimalkv"] + +[tool.setuptools.package-data] +minimalkv = ["py.typed"] [tool.black] exclude = ''' diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 526aeb2..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal = 0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 136559c..0000000 --- a/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python - -from os import path - -from setuptools import find_packages, setup - -here = path.abspath(path.dirname(__file__)) - -with open("README.rst") as f: - long_description = f.read() - -setup( - name="minimalkv", - use_scm_version=True, - setup_requires=["setuptools_scm"], - description=("A key-value storage for binary data, support many " "backends."), - long_description=long_description, - author="Data Engineering Collective", - author_email="minimalkv@uwekorn.com", - url="https://github.com/data-engineering-collective/minimalkv", - license="BSD-3-clause", - packages=find_packages(exclude=["test"]), - install_requires=["uritools"], - python_requires=">=3.8", - package_data={"minimalkv": ["py.typed"]}, - classifiers=[ - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Development Status :: 5 - Production/Stable", - "Operating System :: OS Independent", - ], -) From 359a7cdcb1e6329e0f644c58e7e368aa10b64921 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Thu, 2 Nov 2023 16:44:35 +0100 Subject: [PATCH 03/13] Install ruff pre-commit-hook Install ruff with autofix, list hook at the top. Port settings from flake8 to ruff. --- .flake8 | 11 ----------- .pre-commit-config.yaml | 11 ++++++----- pyproject.toml | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 16 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 8b05742..0000000 --- a/.flake8 +++ /dev/null @@ -1,11 +0,0 @@ -# Taken directly from https://github.com/ambv/black/blob/master/.flake8 -[flake8] -ignore = E203, E266, E501, W503, C901, D104, D100 -max-line-length = 88 -max-complexity = 18 -select = B,C,E,F,W,T4,B9,D -enable-extensions = flake8-docstrings -per-file-ignores = - tests/**:D101,D102,D103,E402 - examples/**:E402 -docstring-convention = numpy \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31e3c37..40f41b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,10 @@ repos: + # Run ruff first because autofix behaviour is enabled + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.3 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/Quantco/pre-commit-mirrors-black rev: 23.7.0 hooks: @@ -6,11 +12,6 @@ repos: args: - --safe - --target-version=py38 - - repo: https://github.com/Quantco/pre-commit-mirrors-flake8 - rev: 6.1.0 - hooks: - - id: flake8-conda - additional_dependencies: [-c, conda-forge, flake8-docstrings=1.5.0, flake8-rst-docstrings=0.0.14] - repo: https://github.com/Quantco/pre-commit-mirrors-isort rev: 5.12.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 0388299..2be4dc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,21 @@ exclude = ''' )/ ''' +[tool.ruff] +line-length = 88 +target-version = "py310" + +ignore = ["E203", "E266", "E501", "C901", "D104", "D100"] +select = ["B", "C", "E", "F", "W", "B9", "D"] + +[tool.ruff.per-file-ignores] +"tests/*" = ["D101", "D102", "D103", "E402"] +"tests/test_azure_store.py" = ["B018"] +"tests/storefact/test_urls.py" = ["C408"] + +[tool.ruff.pydocstyle] +convention = "numpy" + [tool.isort] multi_line_output = 3 include_trailing_comma = true From fe831510f1115c1a00fe93a0764632bf3913eee1 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Fri, 3 Nov 2023 09:47:11 +0100 Subject: [PATCH 04/13] Fix all problems pointed out by ruff --- minimalkv/_boto.py | 2 +- minimalkv/_get_store.py | 6 ++-- minimalkv/_key_value_store.py | 9 ++--- minimalkv/_store_creation.py | 2 +- minimalkv/_urls.py | 16 ++++----- minimalkv/crypt.py | 2 +- minimalkv/db/mongo.py | 4 +-- minimalkv/db/sql.py | 2 +- minimalkv/decorator.py | 3 +- minimalkv/fs.py | 12 +++---- minimalkv/fsspecstore.py | 34 ++++++++----------- minimalkv/git.py | 4 +-- minimalkv/idgen.py | 3 +- minimalkv/memory/redisstore.py | 4 +-- minimalkv/net/_azurestore_new.py | 4 +-- minimalkv/net/_azurestore_old.py | 8 ++--- minimalkv/net/boto3store.py | 18 +++++----- minimalkv/net/botostore.py | 15 ++++---- minimalkv/net/gcstore.py | 3 +- minimalkv/net/s3fsstore.py | 4 +-- tests/basic_store.py | 4 +-- tests/bucket_manager.py | 6 ++-- .../test_creation_boto3store.py | 3 +- tests/test_gcloud_store.py | 2 +- tests/test_prefix_decorator.py | 4 +-- 25 files changed, 76 insertions(+), 98 deletions(-) diff --git a/minimalkv/_boto.py b/minimalkv/_boto.py index cf6dacc..8ab3116 100644 --- a/minimalkv/_boto.py +++ b/minimalkv/_boto.py @@ -35,5 +35,5 @@ def _get_s3bucket( if create_if_missing: return s3con.create_bucket(bucket) else: - raise OSError(f"Bucket {bucket} does not exist") + raise OSError(f"Bucket {bucket} does not exist") from ex raise diff --git a/minimalkv/_get_store.py b/minimalkv/_get_store.py index e9fb4b2..8002c60 100644 --- a/minimalkv/_get_store.py +++ b/minimalkv/_get_store.py @@ -10,8 +10,7 @@ def get_store_from_url( url: str, store_cls: Optional[Type[KeyValueStore]] = None ) -> KeyValueStore: - """ - Take a URL and return a minimalkv store according to the parameters in the URL. + """Take a URL and return a minimalkv store according to the parameters in the URL. Parameters ---------- @@ -97,8 +96,7 @@ def get_store_from_url( def _extract_wrappers(parsed_url: SplitResult) -> List[str]: - """ - Extract wrappers from a parsed URL. + """Extract wrappers from a parsed URL. Wrappers allow you to add additional functionality to a store, e.g. encryption. They can be specified in two ways: diff --git a/minimalkv/_key_value_store.py b/minimalkv/_key_value_store.py index 6f645be..de671ab 100644 --- a/minimalkv/_key_value_store.py +++ b/minimalkv/_key_value_store.py @@ -12,8 +12,7 @@ class KeyValueStore: - """ - Class to access a key-value store. + """Class to access a key-value store. Supported keys are ascii-strings containing alphanumeric characters or symbols out of ``minimalkv._constants.VALID_NON_NUM`` of length not greater than 250. Values @@ -141,8 +140,7 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: raise NotImplementedError def iter_prefixes(self, delimiter: str, prefix: str = "") -> Iterator[str]: - """ - Iterate over unique prefixes in the store up to delimiter, starting with prefix. + """Iterate over unique prefixes in the store up to delimiter, starting with prefix. If ``prefix`` contains ``delimiter``, return the prefix up to the first occurence of delimiter after the prefix. @@ -468,8 +466,7 @@ def __exit__( def _from_parsed_url( cls, parsed_url: SplitResult, query: Dict[str, str] ) -> "KeyValueStore": - """ - Build a ``KeyValueStore`` from a parsed URL. + """Build a ``KeyValueStore`` from a parsed URL. To build a ``KeyValueStore`` from a URL, use :func:`get_store_from_url`. diff --git a/minimalkv/_store_creation.py b/minimalkv/_store_creation.py index 9203981..e11fb5e 100644 --- a/minimalkv/_store_creation.py +++ b/minimalkv/_store_creation.py @@ -50,7 +50,7 @@ def _create_store_gcs(store_type, params): from minimalkv._hstores import HGoogleCloudStore from minimalkv.net.gcstore import GoogleCloudStore - if type(params["credentials"]) is bytes: + if isinstance(params["credentials"], bytes): account_info = json.loads(params["credentials"].decode()) params["credentials"] = Credentials.from_service_account_info( account_info, diff --git a/minimalkv/_urls.py b/minimalkv/_urls.py index 3df619d..c31307f 100644 --- a/minimalkv/_urls.py +++ b/minimalkv/_urls.py @@ -44,14 +44,14 @@ def url2dict(url: str, raise_on_extra_params: bool = False) -> Dict[str, Any]: ) u = urisplit(url) - parsed = dict( - scheme=u.getscheme(), - host=u.gethost(), - port=u.getport(), - path=u.getpath(), - query=u.getquerydict(), - userinfo=u.getuserinfo(), - ) + parsed = { + "scheme": u.getscheme(), + "host": u.gethost(), + "port": u.getport(), + "path": u.getpath(), + "query": u.getquerydict(), + "userinfo": u.getuserinfo(), + } fragment = u.getfragment() params = {"type": parsed["scheme"]} diff --git a/minimalkv/crypt.py b/minimalkv/crypt.py index 7df3d3e..7d564d8 100644 --- a/minimalkv/crypt.py +++ b/minimalkv/crypt.py @@ -128,7 +128,7 @@ def get_file(self, key, file): # noqa D try: f = open(file, "wb") except OSError as e: - raise OSError(f"Error opening {file} for writing: {e!r}") + raise OSError(f"Error opening {file} for writing: {e!r}") from e # file is open, now we call ourself again with a proper file try: diff --git a/minimalkv/db/mongo.py b/minimalkv/db/mongo.py index 6acfbaa..1143b30 100644 --- a/minimalkv/db/mongo.py +++ b/minimalkv/db/mongo.py @@ -34,8 +34,8 @@ def _get(self, key: str) -> bytes: try: item = next(self.db[self.collection].find({"_id": key})) return pickle.loads(item["v"]) - except StopIteration: - raise KeyError(key) + except StopIteration as e: + raise KeyError(key) from e def _open(self, key: str) -> IO: return BytesIO(self._get(key)) diff --git a/minimalkv/db/sql.py b/minimalkv/db/sql.py index f10d76c..e7aaccb 100644 --- a/minimalkv/db/sql.py +++ b/minimalkv/db/sql.py @@ -87,4 +87,4 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: # noqa D query = select(self.table.c.key) if prefix != "": query = query.where(self.table.c.key.like(prefix + "%")) - return map(lambda v: str(v[0]), session.execute(query)) + return (str(v[0]) for v in session.execute(query)) diff --git a/minimalkv/decorator.py b/minimalkv/decorator.py index b7de4c0..553585f 100644 --- a/minimalkv/decorator.py +++ b/minimalkv/decorator.py @@ -139,8 +139,7 @@ def copy(self, source: str, dest: str): # noqa D class PrefixDecorator(KeyTransformingDecorator): - """ - Prefixes any key with a string before passing it on the decorated store. + """Prefixes any key with a string before passing it on the decorated store. Automatically strips the prefix upon key retrieval. diff --git a/minimalkv/fs.py b/minimalkv/fs.py index a0e4b98..86c53a8 100644 --- a/minimalkv/fs.py +++ b/minimalkv/fs.py @@ -82,7 +82,7 @@ def _open(self, key: str) -> IO: return f except OSError as e: if 2 == e.errno: - raise KeyError(key) + raise KeyError(key) from e else: raise @@ -97,7 +97,7 @@ def _copy(self, source: str, dest: str) -> str: return dest except OSError as e: if 2 == e.errno: - raise KeyError(source) + raise KeyError(source) from e else: raise @@ -155,7 +155,7 @@ def keys(self, prefix: str = "") -> List[str]: """ root = os.path.abspath(self.root) result = [] - for dp, dn, fn in os.walk(root): + for dp, _, fn in os.walk(root): for f in fn: key = os.path.join(dp, f)[len(root) + 1 :] if key.startswith(prefix): @@ -174,8 +174,7 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: return iter(self.keys(prefix)) def iter_prefixes(self, delimiter: str, prefix: str = "") -> Iterator[str]: - """ - Iterate over unique prefixes in the store up to delimiter, starting with prefix. + """Iterate over unique prefixes in the store up to delimiter, starting with prefix. If ``prefix`` contains ``delimiter``, return the prefix up to the first occurence of delimiter after the prefix. @@ -224,8 +223,7 @@ def _iter_prefixes_efficient( class WebFilesystemStore(FilesystemStore): - """ - FilesystemStore supporting generating URLS for web applications. + """FilesystemStore supporting generating URLS for web applications. The most common use is to make the ``root`` directory of the filesystem store available through a webserver. diff --git a/minimalkv/fsspecstore.py b/minimalkv/fsspecstore.py index 8f28915..531f664 100644 --- a/minimalkv/fsspecstore.py +++ b/minimalkv/fsspecstore.py @@ -20,8 +20,7 @@ class FSSpecStoreEntry(io.BufferedIOBase): """A file-like object for reading from an entry in an FSSpecStore.""" def __init__(self, file: "AbstractBufferedFile"): - """ - Initialize an FSSpecStoreEntry. + """Initialize an FSSpecStoreEntry. Parameters ---------- @@ -32,8 +31,7 @@ def __init__(self, file: "AbstractBufferedFile"): self._file = file def seek(self, loc: int, whence: int = 0) -> int: - """ - Set current file location. + """Set current file location. Parameters ---------- @@ -46,9 +44,9 @@ def seek(self, loc: int, whence: int = 0) -> int: raise ValueError("I/O operation on closed file.") try: return self._file.seek(loc, whence) - except ValueError: + except ValueError as e: # Map ValueError to IOError - raise OSError + raise OSError from e def tell(self) -> int: """Return the current offset as int. Always >= 0.""" @@ -90,8 +88,7 @@ def __init__( write_kwargs: Optional[dict] = None, custom_fs: Optional["AbstractFileSystem"] = None, ): - """ - Initialize an FSSpecStore. + """Initialize an FSSpecStore. The underlying fsspec FileSystem is created when the store is used for the first time. @@ -126,8 +123,7 @@ def _prefix_exists(self) -> Union[None, bool]: @property def mkdir_prefix(self): - """ - Whether to create the prefix if it does not exist. + """Whether to create the prefix if it does not exist. .. note:: Deprecated in 2.0.0. """ @@ -135,14 +131,14 @@ def mkdir_prefix(self): "The mkdir_prefix attribute is deprecated!" "It will be renamed to _mkdir_prefix in the next release.", DeprecationWarning, + stacklevel=2, ) return self._mkdir_prefix @property def prefix(self): - """ - Get the prefix used on the ``fsspec`` ``FileSystem`` when storing keys. + """Get the prefix used on the ``fsspec`` ``FileSystem`` when storing keys. .. note:: Deprecated in 2.0.0. """ @@ -150,6 +146,7 @@ def prefix(self): "The prefix attribute is deprecated!" "It will be renamed to _prefix in the next release.", DeprecationWarning, + stacklevel=2, ) return self._prefix @@ -184,10 +181,7 @@ def iter_keys(self, prefix: str = "") -> Iterator[str]: all_files_and_dirs = self._fs.find(dir_prefix, prefix=file_prefix) - return map( - lambda k: k.replace(f"{self._prefix}", ""), - all_files_and_dirs, - ) + return (k.replace(f"{self._prefix}", "") for k in all_files_and_dirs) def _delete(self, key: str) -> None: try: @@ -198,16 +192,16 @@ def _delete(self, key: str) -> None: def _open(self, key: str) -> IO: try: return self._fs.open(f"{self._prefix}{key}") - except FileNotFoundError: - raise KeyError(key) + except FileNotFoundError as e: + raise KeyError(key) from e # Required to prevent error when credentials are not sufficient for listing objects def _get_file(self, key: str, file: IO) -> str: try: file.write(self._fs.cat_file(f"{self._prefix}{key}")) return key - except FileNotFoundError: - raise KeyError(key) + except FileNotFoundError as e: + raise KeyError(key) from e def _put_file(self, key: str, file: IO) -> str: self._fs.pipe_file(f"{self._prefix}{key}", file.read(), **self._write_kwargs) diff --git a/minimalkv/git.py b/minimalkv/git.py index 749f392..87830b7 100644 --- a/minimalkv/git.py +++ b/minimalkv/git.py @@ -172,8 +172,8 @@ def _get(self, key: str) -> bytes: fn = self.subdir + "/" + key _, blob_id = tree.lookup_path(self.repo.__getitem__, fn.encode("ascii")) blob = self.repo[blob_id] - except KeyError: - raise KeyError(key) + except KeyError as e: + raise KeyError(key) from e return blob.data diff --git a/minimalkv/idgen.py b/minimalkv/idgen.py index 1a5abbe..10f18ed 100644 --- a/minimalkv/idgen.py +++ b/minimalkv/idgen.py @@ -1,5 +1,4 @@ -""" -In cases where you want to generate IDs automatically, decorators are available. +"""In cases where you want to generate IDs automatically, decorators are available. These should be the outermost decorators, as they change the signature of some of the put methods slightly. diff --git a/minimalkv/memory/redisstore.py b/minimalkv/memory/redisstore.py index 87c1de2..1ace056 100644 --- a/minimalkv/memory/redisstore.py +++ b/minimalkv/memory/redisstore.py @@ -40,9 +40,7 @@ def keys(self, prefix: str = "") -> List[str]: IOError If there was an error accessing the store. """ - return list( - map(lambda b: b.decode(), self.redis.keys(pattern=re.escape(prefix) + "*")) - ) + return [b.decode() for b in self.redis.keys(pattern=re.escape(prefix) + "*")] def iter_keys(self, prefix="") -> Iterator[str]: """Iterate over all keys in the store starting with prefix. diff --git a/minimalkv/net/_azurestore_new.py b/minimalkv/net/_azurestore_new.py index a63b14b..a4e4c50 100644 --- a/minimalkv/net/_azurestore_new.py +++ b/minimalkv/net/_azurestore_new.py @@ -20,8 +20,8 @@ def map_azure_exceptions(key=None, error_codes_pass=()): if error_code is not None and error_code in error_codes_pass: return if error_code == "BlobNotFound": - raise KeyError(key) - raise OSError(str(ex)) + raise KeyError(key) from ex + raise OSError(str(ex)) from ex class AzureBlockBlobStore(KeyValueStore): # noqa D diff --git a/minimalkv/net/_azurestore_old.py b/minimalkv/net/_azurestore_old.py index 610e3c6..6a9b074 100644 --- a/minimalkv/net/_azurestore_old.py +++ b/minimalkv/net/_azurestore_old.py @@ -22,14 +22,14 @@ def map_azure_exceptions(key=None, exc_pass=()): if ex.__class__.__name__ not in exc_pass: s = str(ex) if s.startswith("The specified container does not exist."): - raise OSError(s) - raise KeyError(key) + raise OSError(s) from ex + raise KeyError(key) from ex except AzureHttpError as ex: if ex.__class__.__name__ not in exc_pass: - raise OSError(str(ex)) + raise OSError(str(ex)) from ex except AzureException as ex: if ex.__class__.__name__ not in exc_pass: - raise OSError(str(ex)) + raise OSError(str(ex)) from ex class AzureBlockBlobStore(KeyValueStore): # noqa D diff --git a/minimalkv/net/boto3store.py b/minimalkv/net/boto3store.py index e1afdfd..397bf2f 100644 --- a/minimalkv/net/boto3store.py +++ b/minimalkv/net/boto3store.py @@ -31,8 +31,8 @@ def map_boto3_exceptions(key=None, exc_pass=()): except ClientError as ex: code = ex.response["Error"]["Code"] if code == "404" or code == "NoSuchKey": - raise KeyError(key) - raise OSError(str(ex)) + raise KeyError(key) from ex + raise OSError(str(ex)) from ex class Boto3SimpleKeyFile(io.RawIOBase): # noqa D @@ -120,6 +120,7 @@ def __init__( "The prefix attribute is deprecated and will be removed in the next major release." "Use object_prefix instead.", DeprecationWarning, + stacklevel=2, ) object_prefix = object_prefix or prefix self._object_prefix = object_prefix.strip().lstrip("/") @@ -131,8 +132,7 @@ def __init__( @property def prefix(self) -> str: - """ - Get the prefix used for all keys in this store. + """Get the prefix used for all keys in this store. .. note:: Deprecated in 2.0.0, use :attr:`object_prefix` instead. """ @@ -142,6 +142,7 @@ def prefix(self) -> str: "The `prefix` attribute is deprecated and will be removed in the next major release." "Use `object_prefix` instead.", DeprecationWarning, + stacklevel=2, ) return self._object_prefix @@ -152,9 +153,9 @@ def __new_object(self, name): def iter_keys(self, prefix=""): # noqa D with map_boto3_exceptions(): prefix_len = len(self._object_prefix) - return map( - lambda k: k.key[prefix_len:], - self.bucket.objects.filter(Prefix=self._object_prefix + prefix), + return ( + k.key[prefix_len:] + for k in self.bucket.objects.filter(Prefix=self._object_prefix + prefix) ) def _delete(self, key): @@ -247,8 +248,7 @@ def _url_for(self, key): ) def __eq__(self, other): - """ - Assert that two ``Boto3Store``s are equal. + """Assert that two ``Boto3Store``s are equal. The bucket name and other configuration parameters are compared. See :func:`from_url` for details on the connection parameters. diff --git a/minimalkv/net/botostore.py b/minimalkv/net/botostore.py index af646cc..b05a7c6 100644 --- a/minimalkv/net/botostore.py +++ b/minimalkv/net/botostore.py @@ -13,11 +13,11 @@ def map_boto_exceptions(key=None, exc_pass=()): yield except StorageResponseError as e: if e.code == "NoSuchKey": - raise KeyError(key) - raise OSError(str(e)) + raise KeyError(key) from e + raise OSError(str(e)) from e except (BotoClientError, BotoServerError) as e: if e.__class__.__name__ not in exc_pass: - raise OSError(str(e)) + raise OSError(str(e)) from e class BotoStore(KeyValueStore, UrlMixin, CopyMixin): # noqa D @@ -46,8 +46,7 @@ def __new_key(self, name): return k def __upload_args(self) -> Dict[str, str]: - """ - Generate a dictionary of arguments to pass to ``set_content_from`` functions. + """Generate a dictionary of arguments to pass to ``set_content_from`` functions. This allows us to save API calls by passing the necessary parameters on with the upload. @@ -76,9 +75,7 @@ def iter_keys(self, prefix="") -> Iterator[str]: """ with map_boto_exceptions(): prefix_len = len(self.prefix) - return map( - lambda k: k.name[prefix_len:], self.bucket.list(self.prefix + prefix) - ) + return (k.name[prefix_len:] for k in self.bucket.list(self.prefix + prefix)) def _has_key(self, key: str) -> bool: with map_boto_exceptions(key=key): @@ -91,7 +88,7 @@ def _delete(self, key: str) -> None: self.bucket.delete_key(self.prefix + key) except StorageResponseError as e: if e.code != "NoSuchKey": - raise OSError(str(e)) + raise OSError(str(e)) from e def _get(self, key: str) -> bytes: k = self.__new_key(key) diff --git a/minimalkv/net/gcstore.py b/minimalkv/net/gcstore.py index 9fa44db..a820303 100644 --- a/minimalkv/net/gcstore.py +++ b/minimalkv/net/gcstore.py @@ -42,7 +42,8 @@ def __init__( This was caused by the following error: {error} - """ + """, + stacklevel=2, ) self._credentials = credentials diff --git a/minimalkv/net/s3fsstore.py b/minimalkv/net/s3fsstore.py index d091314..28ff2c3 100644 --- a/minimalkv/net/s3fsstore.py +++ b/minimalkv/net/s3fsstore.py @@ -88,8 +88,8 @@ def _url_for(self, key) -> str: def _from_parsed_url( cls, parsed_url: SplitResult, query: Dict[str, str] ) -> "S3FSStore": # noqa D - """ - Build an ``S3FSStore`` from a parsed URL. + """Build an ``S3FSStore`` from a parsed URL. + To build an ``S3FSStore`` from a URL, use :func:`get_store_from_url`. URl format: diff --git a/tests/basic_store.py b/tests/basic_store.py index 20798c2..cdb8563 100644 --- a/tests/basic_store.py +++ b/tests/basic_store.py @@ -431,12 +431,12 @@ def test_uuid_decorator(self, ustore, value): def test_advertises_ttl_features(self, store): assert store.ttl_support is True assert hasattr(store, "ttl_support") - assert getattr(store, "ttl_support") is True + assert store.ttl_support is True def test_advertises_ttl_features_through_decorator(self, dstore): assert dstore.ttl_support is True assert hasattr(dstore, "ttl_support") - assert getattr(dstore, "ttl_support") is True + assert dstore.ttl_support is True def test_can_pass_ttl_through_decorator(self, dstore, key, value): dstore.put(key, value, ttl_secs=10) diff --git a/tests/bucket_manager.py b/tests/bucket_manager.py index c2f8f78..2a150dc 100644 --- a/tests/bucket_manager.py +++ b/tests/bucket_manager.py @@ -55,8 +55,7 @@ def boto3_bucket( is_secure=None, **kwargs, ): - """ - Create a boto3 bucket. + """Create a boto3 bucket. The bucket is deleted after the consuming function returns. """ @@ -85,8 +84,7 @@ def boto3_bucket_reference( port=None, is_secure=None, ): - """ - Create a boto3 bucket reference. + """Create a boto3 bucket reference. The bucket is not created. """ diff --git a/tests/store_creation/test_creation_boto3store.py b/tests/store_creation/test_creation_boto3store.py index 3043679..525a08f 100644 --- a/tests/store_creation/test_creation_boto3store.py +++ b/tests/store_creation/test_creation_boto3store.py @@ -42,8 +42,7 @@ def test_equal_access(): def s3fsstores_equal(store1, store2): - """ - Return whether two ``S3FSStore``s are equal. + """Return whether two ``S3FSStore``s are equal. The bucket name and other configuration parameters are compared. See :func:`from_url` for details on the connection parameters. diff --git a/tests/test_gcloud_store.py b/tests/test_gcloud_store.py index 1db23e3..f171c5b 100644 --- a/tests/test_gcloud_store.py +++ b/tests/test_gcloud_store.py @@ -87,7 +87,7 @@ def get_bucket_from_store(gcstore: GoogleCloudStore) -> Bucket: def get_client_from_store(gcstore: GoogleCloudStore) -> Client: - if type(gcstore._credentials) is str: + if isinstance(gcstore._credentials, str): client = Client.from_service_account_json(gcstore._credentials) else: client = Client(credentials=gcstore._credentials, project=gcstore.project_name) diff --git a/tests/test_prefix_decorator.py b/tests/test_prefix_decorator.py index eae5fa7..0a1b3dc 100644 --- a/tests/test_prefix_decorator.py +++ b/tests/test_prefix_decorator.py @@ -48,7 +48,7 @@ def test_put_returns_correct_key(self, store, prefix, key, value): def test_put_sets_prefix(self, store, prefix, key, value): full_key = prefix + key - key == store.put(key, value) + assert key == store.put(key, value) assert store._dstore.get(full_key) == value @@ -57,7 +57,7 @@ def test_put_file_returns_correct_key(self, store, prefix, key, value): def test_put_file_sets_prefix(self, store, prefix, key, value): full_key = prefix + key - key == store.put_file(key, BytesIO(value)) + assert key == store.put_file(key, BytesIO(value)) assert store._dstore.get(full_key) == value From 089faf1cb05b377b1c48e985f4de6bc3204ccd60 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Fri, 3 Nov 2023 10:09:22 +0100 Subject: [PATCH 05/13] Add auto update workflow for pre-commit-hooks --- .github/workflows/pre-commit-autoupdate.yml | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/pre-commit-autoupdate.yml diff --git a/.github/workflows/pre-commit-autoupdate.yml b/.github/workflows/pre-commit-autoupdate.yml new file mode 100644 index 0000000..fc4e33f --- /dev/null +++ b/.github/workflows/pre-commit-autoupdate.yml @@ -0,0 +1,45 @@ +name: pre-commit autoupdate +on: + workflow_dispatch: + schedule: + - cron: "0 6 4 * *" + +defaults: + run: + shell: bash -el {0} + +jobs: + check_update: + name: Check if newer version exists + runs-on: ubuntu-latest + steps: + - name: Checkout branch + uses: actions/checkout@v4 + # We need to checkout with SSH here to have actions run on the PR. + with: + ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} + - name: Set up Conda env + uses: mamba-org/setup-micromamba@db1df3ba9e07ea86f759e98b575c002747e9e757 + with: + environment-name: pre-commit + create-args: >- + -c + conda-forge + pre-commit + mamba + - name: Update pre-commit hooks and run + id: versions + env: + PRE_COMMIT_USE_MAMBA: 1 + run: | + pre-commit autoupdate + pre-commit run -a || true + - uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 + with: + commit-message: "Auto-update pre-commit hooks" + title: "Auto-update pre-commit hooks" + body: | + New versions of the used pre-commit hooks were detected. + This PR updates them to the latest and already ran `pre-commit run -a` for you to fix any changes in formatting. + branch: pre-commit-autoupdate + delete-branch: true From 05fd193e767c84f333a53435cde6f272f6f91308 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Fri, 3 Nov 2023 11:12:15 +0100 Subject: [PATCH 06/13] Replace setuptools with pip --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 82f1f35..4a018c3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,5 +10,5 @@ conda: python: install: - - method: setuptools + - method: pip path: . From 4cab6af3afc9370bcdfc1b0d42dbed2af02bfb63 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Fri, 3 Nov 2023 11:51:12 +0100 Subject: [PATCH 07/13] Address issue #44: determine version programmatically. The version for the documentation build was hard-coded in . It is now determined programmtically. --- docs/conf.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 10b297b..4fe8231 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,6 @@ import inspect import os +import re import sys from sphinx.ext import apidoc @@ -13,6 +14,37 @@ os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())) # type: ignore ) + +def simplify_version(version): + """ + Simplifies the version string to only include the major.minor.patch components. + + Example: + '1.8.2.post0+g476bc9e.d20231103' -> '1.8.2' + """ + match = re.match(r"^(\d+\.\d+\.\d+)(?:\.post\d+)?", version) + return match.group(1) if match else version + + +try: + import minimalkv + + version = simplify_version(minimalkv.__version__) +except ImportError: + import pkg_resources + + version = simplify_version(pkg_resources.get_distribution("minimalkv").version) + +print(f"Building docs for version: {version}") + +# The version info is fetched programmatically. It acts as replacement for +# |version| and |release|, it is also used in various other places throughout +# the built documents. +# +# major.minor.patch + +release = version + # Generate module references output_dir = os.path.abspath(os.path.join(__location__, "../docs/_rst")) module_dir = os.path.abspath(os.path.join(__location__, "..", package)) @@ -37,14 +69,6 @@ project = "minimalkv" copyright = "2011-2021, The minimalkv contributors" -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = "0.14.1" -# The full version, including alpha/beta/rc tags. -release = "0.14.1" exclude_trees = ["_build"] From dcdff8589596b538ae33de1adf4bc9dc5357b85a Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Fri, 3 Nov 2023 11:51:50 +0100 Subject: [PATCH 08/13] Update changes.rst --- docs/changes.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index f86f4a7..85854b2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,14 @@ Changelog ********* +1.8.2 +===== +* Include Python 3.12 in CI +* Migrate setup.cfg and setup.py into pyproject.toml +* Port to ruff +* Include pre-commit autoupdate workflow +* Determine version in ``docs/conf.py`` automatically + 1.8.1 ===== * Drop `pkg_resources` and use `importlib.metadata` to access package version string. From ce5370879b983da5f78e3f9fe9f5656e8e2ee4d7 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Fri, 3 Nov 2023 16:33:31 +0100 Subject: [PATCH 09/13] Simplify code in docs/conf.py. Prettifying the version isn't necessary when a tag is assigned correctly. --- docs/conf.py | 35 ++++------------------------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 4fe8231..1528f97 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,13 @@ import inspect import os -import re import sys from sphinx.ext import apidoc +from minimalkv import __version__ as version + +release = version + sys.path.append("../") package = "minimalkv" @@ -15,36 +18,6 @@ ) -def simplify_version(version): - """ - Simplifies the version string to only include the major.minor.patch components. - - Example: - '1.8.2.post0+g476bc9e.d20231103' -> '1.8.2' - """ - match = re.match(r"^(\d+\.\d+\.\d+)(?:\.post\d+)?", version) - return match.group(1) if match else version - - -try: - import minimalkv - - version = simplify_version(minimalkv.__version__) -except ImportError: - import pkg_resources - - version = simplify_version(pkg_resources.get_distribution("minimalkv").version) - -print(f"Building docs for version: {version}") - -# The version info is fetched programmatically. It acts as replacement for -# |version| and |release|, it is also used in various other places throughout -# the built documents. -# -# major.minor.patch - -release = version - # Generate module references output_dir = os.path.abspath(os.path.join(__location__, "../docs/_rst")) module_dir = os.path.abspath(os.path.join(__location__, "..", package)) From 5c40e293b9e7aa14cf9dd5ed4780cbbecca70af9 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Mon, 6 Nov 2023 09:22:42 +0100 Subject: [PATCH 10/13] Install ruff formatter --- .pre-commit-config.yaml | 14 +++----------- pyproject.toml | 15 ++++----------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40f41b9..c14acc9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,18 +5,10 @@ repos: hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - - repo: https://github.com/Quantco/pre-commit-mirrors-black - rev: 23.7.0 - hooks: - - id: black-conda - args: - - --safe - - --target-version=py38 - - repo: https://github.com/Quantco/pre-commit-mirrors-isort - rev: 5.12.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.4 hooks: - - id: isort-conda - additional_dependencies: [-c, conda-forge, toml=0.10.2] + - id: ruff-format - repo: https://github.com/Quantco/pre-commit-mirrors-mypy rev: "1.5.1" hooks: diff --git a/pyproject.toml b/pyproject.toml index 2be4dc4..4d5e3c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,21 +35,11 @@ include = ["minimalkv"] [tool.setuptools.package-data] minimalkv = ["py.typed"] -[tool.black] -exclude = ''' -/( - \.eggs - | \.git - | \.venv - | build - | dist -)/ -''' - [tool.ruff] line-length = 88 target-version = "py310" +[tool.ruff.lint] ignore = ["E203", "E266", "E501", "C901", "D104", "D100"] select = ["B", "C", "E", "F", "W", "B9", "D"] @@ -61,6 +51,9 @@ select = ["B", "C", "E", "F", "W", "B9", "D"] [tool.ruff.pydocstyle] convention = "numpy" +[tool.ruff.format] +quote-style = "double" + [tool.isort] multi_line_output = 3 include_trailing_comma = true From ebe66d47f105dcd06ba6a64d05c3cc763a91e20a Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Mon, 6 Nov 2023 10:43:35 +0100 Subject: [PATCH 11/13] Update config to use ruff isort --- docs/conf.py | 3 ++- minimalkv/_boto.py | 7 +++++-- pyproject.toml | 18 ++++++++---------- tests/test_filesystem_store.py | 8 +++++--- tests/test_get_store_from_url.py | 6 ++++-- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1528f97..b19f5b4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,8 @@ html_theme = "alabaster" __location__ = os.path.join( - os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())) # type: ignore + os.getcwd(), + os.path.dirname(inspect.getfile(inspect.currentframe())), # type: ignore ) diff --git a/minimalkv/_boto.py b/minimalkv/_boto.py index 8ab3116..5abbc58 100644 --- a/minimalkv/_boto.py +++ b/minimalkv/_boto.py @@ -7,8 +7,11 @@ def _get_s3bucket( create_if_missing=True, ): # TODO: Write docstring. - from boto.s3.connection import S3ResponseError # type: ignore - from boto.s3.connection import OrdinaryCallingFormat, S3Connection + from boto.s3.connection import ( # type: ignore + OrdinaryCallingFormat, + S3Connection, + S3ResponseError, + ) s3_connection_params = { "aws_access_key_id": access_key, diff --git a/pyproject.toml b/pyproject.toml index 4d5e3c6..eea2c98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,11 +37,16 @@ minimalkv = ["py.typed"] [tool.ruff] line-length = 88 -target-version = "py310" +target-version = "py38" [tool.ruff.lint] ignore = ["E203", "E266", "E501", "C901", "D104", "D100"] -select = ["B", "C", "E", "F", "W", "B9", "D"] +select = ["B", "C", "E", "F", "W", "B9", "D", "I"] + +[tool.ruff.lint.isort] +force-wrap-aliases = true +combine-as-imports = true +known-first-party = ["minimalkv"] [tool.ruff.per-file-ignores] "tests/*" = ["D101", "D102", "D103", "E402"] @@ -53,14 +58,7 @@ convention = "numpy" [tool.ruff.format] quote-style = "double" - -[tool.isort] -multi_line_output = 3 -include_trailing_comma = true -line_length = 88 -known_first_party = "minimalkv" -skip_glob = '\.eggs/*,\.git/*,\.venv/*,build/*,dist/*' -default_section = 'THIRDPARTY' +indent-style = "space" [tool.mypy] python_version = 3.8 diff --git a/tests/test_filesystem_store.py b/tests/test_filesystem_store.py index d8350ad..1901b0e 100644 --- a/tests/test_filesystem_store.py +++ b/tests/test_filesystem_store.py @@ -3,9 +3,11 @@ import tempfile from io import BytesIO from unittest.mock import Mock -from urllib.parse import quote as url_quote -from urllib.parse import unquote as url_unquote -from urllib.parse import urlparse +from urllib.parse import ( + quote as url_quote, + unquote as url_unquote, + urlparse, +) import pytest from basic_store import BasicStore diff --git a/tests/test_get_store_from_url.py b/tests/test_get_store_from_url.py index e1a072e..25a3cc8 100644 --- a/tests/test_get_store_from_url.py +++ b/tests/test_get_store_from_url.py @@ -2,8 +2,10 @@ import pytest -from minimalkv._get_store import get_store -from minimalkv._get_store import get_store_from_url as get_store_from_url_new +from minimalkv._get_store import ( + get_store, + get_store_from_url as get_store_from_url_new, +) from minimalkv._hstores import HDictStore from minimalkv._key_value_store import KeyValueStore from minimalkv._urls import url2dict From 9398e850559330aa4f5116e16cf3d94a5ffe9bca Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Mon, 6 Nov 2023 11:29:43 +0100 Subject: [PATCH 12/13] Install qc version of hook --- .pre-commit-config.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c14acc9..d07a520 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,13 @@ repos: # Run ruff first because autofix behaviour is enabled - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.3 + - repo: https://github.com/Quantco/pre-commit-mirrors-ruff + rev: "0.1.3" hooks: - - id: ruff - args: [--fix, --exit-non-zero-on-fix] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.4 + - id: ruff-conda + - repo: https://github.com/Quantco/pre-commit-mirrors-ruff + rev: "0.1.3" hooks: - - id: ruff-format + - id: ruff-format-conda - repo: https://github.com/Quantco/pre-commit-mirrors-mypy rev: "1.5.1" hooks: From 0d2d76629c837016db586dc0d14830a455981029 Mon Sep 17 00:00:00 2001 From: Thomas Marwitz Date: Mon, 6 Nov 2023 13:16:06 +0100 Subject: [PATCH 13/13] Unify ruff hooks in single section --- .pre-commit-config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d07a520..be34607 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,9 +4,6 @@ repos: rev: "0.1.3" hooks: - id: ruff-conda - - repo: https://github.com/Quantco/pre-commit-mirrors-ruff - rev: "0.1.3" - hooks: - id: ruff-format-conda - repo: https://github.com/Quantco/pre-commit-mirrors-mypy rev: "1.5.1"