Add ruff pre-commit hook (#1502)

.pre-commit-config.yaml

@@ -16,6 +16,13 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.1.13
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
   - repo: https://github.com/PyCQA/flake8
     rev: 5.0.4
     hooks:

fsspec/exceptions.py

@@ -10,12 +10,8 @@ class BlocksizeMismatchError(ValueError):
     written with
     """
 
-    ...
-
 
 class FSTimeoutError(asyncio.TimeoutError):
     """
     Raised when a fsspec function timed out occurs
     """
-
-    ...

fsspec/gui.py

@@ -153,8 +153,9 @@ def _emit(self, sig, value=None):
                         break
                 except Exception as e:
                     logger.exception(
-                        "Exception (%s) while executing callback for signal: %s"
-                        "" % (e, sig)
+                        "Exception (%s) while executing callback for signal: %s",
+                        e,
+                        sig,
                     )
 
     def show(self, threads=False):

fsspec/implementations/cache_mapper.py

@@ -2,13 +2,9 @@
 
 import abc
 import hashlib
-from typing import TYPE_CHECKING
 
 from fsspec.implementations.local import make_path_posix
 
-if TYPE_CHECKING:
-    from typing import Any
-
 
 class AbstractCacheMapper(abc.ABC):
     """Abstract super-class for mappers from remote URLs to local cached
@@ -19,7 +15,7 @@ class AbstractCacheMapper(abc.ABC):
     def __call__(self, path: str) -> str:
         ...
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         # Identity only depends on class. When derived classes have attributes
         # they will need to be included.
         return isinstance(other, type(self))
@@ -56,7 +52,7 @@ def __call__(self, path: str) -> str:
         else:
             return prefix  # No separator found, simple filename
 
-    def __eq__(self, other: Any) -> bool:
+    def __eq__(self, other: object) -> bool:
         return super().__eq__(other) and self.directory_levels == other.directory_levels
 
     def __hash__(self) -> int:

fsspec/implementations/http.py

@@ -251,7 +251,7 @@ async def _get_file(
             if isfilelike(lpath):
                 outfile = lpath
             else:
-                outfile = open(lpath, "wb")
+                outfile = open(lpath, "wb")  # noqa: ASYNC101
 
             try:
                 chunk = True
@@ -279,7 +279,7 @@ async def gen_chunks():
                 context = nullcontext(lpath)
                 use_seek = False  # might not support seeking
             else:
-                context = open(lpath, "rb")
+                context = open(lpath, "rb")  # noqa: ASYNC101
                 use_seek = True
 
             with context as f:
@@ -801,7 +801,7 @@ async def get_range(session, url, start, end, file=None, **kwargs):
     async with r:
         out = await r.read()
     if file:
-        with open(file, "r+b") as f:
+        with open(file, "r+b") as f:  # noqa: ASYNC101
             f.seek(start)
             f.write(out)
     else:

fsspec/implementations/tests/test_cached.py

@@ -1022,7 +1022,7 @@ def test_multi_cache(protocol):
 def test_multi_cat(protocol, ftp_writable):
     host, port, user, pw = ftp_writable
     fs = FTPFileSystem(host, port, user, pw)
-    for fn in {"/file0", "/file1"}:
+    for fn in ("/file0", "/file1"):
         with fs.open(fn, "wb") as f:
             f.write(b"hello")
 

fsspec/implementations/tests/test_dirfs.py

@@ -355,7 +355,7 @@ async def _walk(path, *args, **kwargs):
 
     actual = []
     async for entry in adirfs._walk("root", *ARGS, **KWARGS):
-        actual.append(entry)
+        actual.append(entry)  # noqa: PERF402
     assert actual == [("root", ["foo", "bar"], ["baz", "qux"])]
     adirfs.fs._walk.assert_called_once_with(f"{PATH}/root", *ARGS, **KWARGS)
 

fsspec/implementations/tests/test_http.py

@@ -557,7 +557,7 @@ async def test_async_walk(server):
     # No maxdepth
     res = []
     async for a in fs._walk(server + "/index"):
-        res.append(a)
+        res.append(a)  # noqa: PERF402
     assert res == [(server + "/index", [], ["realfile"])]
 
     # maxdepth=0

fsspec/implementations/tests/test_reference.py

@@ -61,9 +61,9 @@ def test_nested_dirs_ls():
     refs = {"a": "A", "B/C/b": "B", "B/C/d": "d", "B/_": "_"}
     fs = fsspec.filesystem("reference", fo=refs)
     assert len(fs.ls("")) == 2
-    assert set(e["name"] for e in fs.ls("")) == set(["a", "B"])
+    assert {e["name"] for e in fs.ls("")} == {"a", "B"}
     assert len(fs.ls("B")) == 2
-    assert set(e["name"] for e in fs.ls("B")) == set(["B/C", "B/_"])
+    assert {e["name"] for e in fs.ls("B")} == {"B/C", "B/_"}
 
 
 def test_info(server):  # noqa: F811

fsspec/implementations/tests/test_tar.py

@@ -42,7 +42,7 @@ def test_info():
             assert lhs == expected
 
         # Iterate over all files.
-        for f, v in archive_data.items():
+        for f in archive_data:
             lhs = fs.info(f)
 
             # Probe some specific fields of Tar archives.

fsspec/implementations/tests/test_zip.py

@@ -12,7 +12,7 @@ def test_info():
         fs = fsspec.filesystem("zip", fo=z)
 
         # Iterate over all files.
-        for f, v in archive_data.items():
+        for f in archive_data:
             lhs = fs.info(f)
 
             # Probe some specific fields of Zip archives.

fsspec/parquet.py

@@ -131,10 +131,8 @@ def open_parquet_file(
         cache_type="parts",
         cache_options={
             **options,
-            **{
-                "data": data.get(fn, {}),
-                "strict": strict,
-            },
+            "data": data.get(fn, {}),
+            "strict": strict,
         },
         **kwargs,
     )

fsspec/spec.py

@@ -1398,7 +1398,9 @@ def to_json(self):
         )
         return json.dumps(
             dict(
-                **{"cls": cls, "protocol": proto, "args": self.storage_args},
+                cls=cls,
+                protocol=proto,
+                args=self.storage_args,
                 **self.storage_options,
             )
         )

fsspec/tests/test_fuse.py

@@ -123,6 +123,7 @@ def test_chmod(mount_local):
         ["cp", str(mount_dir / "text"), str(mount_dir / "new")],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
+        check=False,
     )
 
     assert cp.stderr == b""

fsspec/tests/test_spec.py

@@ -1147,6 +1147,7 @@ def test_posix_tests_bash_stat(path, expected, glob_files_folder):
             f"cd {glob_files_folder} && shopt -s globstar && stat -c %N {bash_path}",
         ],
         capture_output=True,
+        check=False,
     )
     # Remove the last element always empty
     bash_output = bash_output.stdout.decode("utf-8").replace("'", "").split("\n")[:-1]

pyproject.toml

@@ -22,6 +22,59 @@ exclude = '''
 )
 '''
 
+[tool.ruff]
+exclude = [
+    ".tox",
+    "build",
+    "docs/source/conf.py",
+    "versioneer.py",
+    "fsspec/_version",
+]
+select = [
+    # fix noqas in fsspec/implementations/http.py
+    "ASYNC",
+    # "B", enable in later PR
+    "C4",
+    "G",
+    "E4",
+    "E7",
+    "E9",
+    "F",
+    "PERF",
+    "PLC",
+    "PLE",
+    "PLR1722",
+    "PLW1510",
+    "PLW3301",
+    "PIE800",
+    "PIE804",
+    "PIE807",
+    "PIE810",
+    # "PT", enable in later PR
+    "PYI",
+    "RUF006",
+    "SLOT",
+    "SIM101",
+]
+line-length = 88
+ignore = [
+    # Assigning lambda expression
+    "E731",
+    # Ambiguous variable names
+    "E741",
+    # line break before binary operator
+    # uncomment when implemented in ruff
+    # "W503",
+    # whitespace before :
+    "E203",
+    # redefs
+    "F811",
+    # Fix these codes later
+    "G004",
+    "PERF203",
+    "PERF401",
+]
+
 [tool.pytest.ini_options]
 # custom markers, need to be defined to avoid warnings
 markers = [