chore(dev-deps): replace flake8 et al with ruff and fix lints

Replace flake8 and friends with `ruff`.

BREAKING CHANGE: Expression types have been removed from `ibis.expr.api`. Use `import ibis.expr.types as ir` to access these types.

.pre-commit-config.yaml

@@ -25,26 +25,16 @@ repos:
     rev: 22.12.0
     hooks:
       - id: black
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.189
     hooks:
-      - id: flake8
-        exclude: (^ibis/tests/sql/snapshots/|_py310\.py$)
+      - id: ruff
+        # makes sure to exclude a file even if it's passed in explicitly
+        args: ["--force-exclude"]
   - repo: https://github.com/pycqa/docformatter
     rev: v1.5.1
     hooks:
       - id: docformatter
-  - repo: https://github.com/MarcoGorelli/absolufy-imports
-    rev: v0.3.1
-    hooks:
-      - id: absolufy-imports
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
-    hooks:
-      - id: pyupgrade
-        entry: pyupgrade --py38-plus --keep-runtime-typing
-        types:
-          - python
   - repo: local
     hooks:
       - id: prettier
@@ -111,6 +101,5 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
-      - id: debug-statements
       - id: check-executables-have-shebangs
       - id: check-shebang-scripts-are-executable

ci/make_geography_db.py

@@ -110,7 +110,7 @@ def main() -> None:
     db_path = Path(args.output_directory).joinpath("geography.db")
     con = sa.create_engine(f"sqlite:///{db_path}")
     make_geography_db(input_data, con)
-    print(db_path)
+    print(db_path)  # noqa: T201
 
 
 if __name__ == "__main__":

ibis/__init__.py

@@ -7,7 +7,7 @@
 from ibis.config import options
 from ibis.expr import api
 from ibis.expr import types as ir
-from ibis.expr.api import *  # noqa: F401,F403
+from ibis.expr.api import *  # noqa: F403
 
 __all__ = ['api', 'ir', 'util', 'BaseBackend', 'IbisError', 'options']
 __all__ += api.__all__

ibis/backends/base/__init__.py

@@ -178,15 +178,15 @@ def __init__(self, backend: BaseBackend):
     def __getitem__(self, name) -> ir.Table:
         try:
             return self._backend.table(name)
-        except Exception as exc:
+        except Exception as exc:  # noqa: BLE001
             raise KeyError(name) from exc
 
     def __getattr__(self, name) -> ir.Table:
         if name.startswith("_"):
             raise AttributeError(name)
         try:
             return self._backend.table(name)
-        except Exception as exc:
+        except Exception as exc:  # noqa: BLE001
             raise AttributeError(name) from exc
 
     def __iter__(self) -> Iterator[str]:
@@ -217,7 +217,7 @@ def _import_pyarrow():
             import pyarrow
         except ImportError:
             raise ModuleNotFoundError(
-                "Exporting to arrow formats requires `pyarrow` but it is not installed"  # noqa: ignore
+                "Exporting to arrow formats requires `pyarrow` but it is not installed"
             )
         else:
             return pyarrow

ibis/backends/base/sql/__init__.py

@@ -135,6 +135,7 @@ def raw_sql(self, query: str) -> Any:
         if cursor:
             return cursor
         cursor.release()
+        return None
 
     @contextlib.contextmanager
     def _safe_raw_sql(self, *args, **kwargs):

ibis/backends/base/sql/compiler/query_builder.py

@@ -279,7 +279,7 @@ def compile(self):
 
     def format_subqueries(self):
         if not self.subqueries:
-            return
+            return None
 
         context = self.context
 

ibis/backends/base/sql/ddl.py

@@ -11,7 +11,7 @@
 
 def _sanitize_format(format):
     if format is None:
-        return
+        return None
     format = format.upper()
     format = _format_aliases.get(format, format)
     if format not in ('PARQUET', 'AVRO', 'TEXTFILE'):

ibis/backends/base/sql/registry/string.py

@@ -36,6 +36,8 @@ def string_find(translator, op):
             return 'locate({}, {}, {}) - 1'.format(
                 substr_formatted, arg_formatted, sval + 1
             )
+        else:
+            raise ValueError(f"invalid `start` value: {sval}")
     else:
         return f'locate({substr_formatted}, {arg_formatted}) - 1'
 

ibis/backends/bigquery/tests/unit/udf/test_core.py

@@ -64,7 +64,7 @@ def test_assign(snapshot):
     def f():
         a = 1
         a = 2
-        print(a)
+        print(a)  # noqa: T201
         return 1
 
     js = compile(f)

ibis/backends/bigquery/udf/core.py

@@ -560,9 +560,9 @@ def range(n):
         some_stuff = [x + y for x, y in [[1, 4], [2, 5], [3, 6]] if 2 < x < 3]
         some_stuff1 = [range(x) for x in [1, 2, 3]]
         some_stuff2 = [x + y for x, y in [(1, 4), (2, 5), (3, 6)]]
-        print(some_stuff)
-        print(some_stuff1)
-        print(some_stuff2)
+        print(some_stuff)  # noqa: T201
+        print(some_stuff1)  # noqa: T201
+        print(some_stuff2)  # noqa: T201
 
         x = 1
         y = 2
@@ -591,7 +591,7 @@ def range(n):
         w = 3
         w = not False
         yyz = None
-        print(yyz)
+        print(yyz)  # noqa: T201
         foobar = x < y < z < w  # x < y and y < z
         foobar = 1
         baz = foobar // 3
@@ -604,4 +604,4 @@ def range(n):
         nnn = len(values)
         return [sum(values) - a + b * y**-x, z, foo.width, nnn]
 
-    print(my_func.js)
+    print(my_func.js)  # noqa: T201

ibis/backends/clickhouse/__init__.py

@@ -181,7 +181,7 @@ def do_connect(
         >>> client = ibis.clickhouse.connect(host=clickhouse_host,  port=clickhouse_port)
         >>> client  # doctest: +ELLIPSIS
         <ibis.clickhouse.client.ClickhouseClient object at 0x...>
-        """  # noqa: E501
+        """
         options = dict(
             host=host,
             port=port,

ibis/backends/clickhouse/tests/test_client.py

@@ -176,7 +176,7 @@ def test_insert_with_more_columns(temporary_alltypes, df):
             ibis.schema(dict(a=dt.UInt8(nullable=False), b=dt.UInt16(nullable=False))),
         ),
         (
-            "SELECT string_col, sum(double_col) as b FROM functional_alltypes GROUP BY string_col",  # noqa: E501
+            "SELECT string_col, sum(double_col) as b FROM functional_alltypes GROUP BY string_col",
             ibis.schema(
                 dict(
                     string_col=dt.String(nullable=True),

ibis/backends/clickhouse/tests/test_operators.py

@@ -220,7 +220,7 @@ def test_simple_case(con, alltypes, translate):
         t.string_col.case().when('foo', 'bar').when('baz', 'qux').else_('default').end()
     )
 
-    expected = """CASE string_col WHEN 'foo' THEN 'bar' WHEN 'baz' THEN 'qux' ELSE 'default' END"""  # noqa: E501
+    expected = """CASE string_col WHEN 'foo' THEN 'bar' WHEN 'baz' THEN 'qux' ELSE 'default' END"""
     assert translate(expr.op()) == expected
     assert len(con.execute(expr))
 
@@ -235,7 +235,7 @@ def test_search_case(con, alltypes, translate):
         .end()
     )
 
-    expected = """CASE WHEN float_col > 0 THEN int_col * 2 WHEN float_col < 0 THEN int_col ELSE 0 END"""  # noqa: E501
+    expected = """CASE WHEN float_col > 0 THEN int_col * 2 WHEN float_col < 0 THEN int_col ELSE 0 END"""
     assert translate(expr.op()) == expected
     assert len(con.execute(expr))
 

ibis/backends/conftest.py

@@ -484,6 +484,7 @@ def _setup_backend(
             "windows prevents two connections to the same duckdb file "
             "even in the same process"
         )
+        return None
     else:
         cls = _get_backend_conf(backend)
         return cls.load_data(

ibis/backends/dask/execution/__init__.py

@@ -1,14 +1,14 @@
-from ibis.backends.dask.execution.aggregations import *  # noqa: F401,F403
-from ibis.backends.dask.execution.arrays import *  # noqa: F401,F403
-from ibis.backends.dask.execution.decimal import *  # noqa: F401,F403
-from ibis.backends.dask.execution.generic import *  # noqa: F401,F403
-from ibis.backends.dask.execution.indexing import *  # noqa: F401,F403
-from ibis.backends.dask.execution.join import *  # noqa: F401,F403
-from ibis.backends.dask.execution.maps import *  # noqa: F401,F403
-from ibis.backends.dask.execution.numeric import *  # noqa: F401,F403
-from ibis.backends.dask.execution.reductions import *  # noqa: F401,F403
-from ibis.backends.dask.execution.selection import *  # noqa: F401,F403
-from ibis.backends.dask.execution.strings import *  # noqa: F401,F403
-from ibis.backends.dask.execution.structs import *  # noqa: F401,F403
-from ibis.backends.dask.execution.temporal import *  # noqa: F401,F403
-from ibis.backends.dask.execution.window import *  # noqa: F401,F403
+from ibis.backends.dask.execution.aggregations import *  # noqa: F403
+from ibis.backends.dask.execution.arrays import *  # noqa: F403
+from ibis.backends.dask.execution.decimal import *  # noqa: F403
+from ibis.backends.dask.execution.generic import *  # noqa: F403
+from ibis.backends.dask.execution.indexing import *  # noqa: F403
+from ibis.backends.dask.execution.join import *  # noqa: F403
+from ibis.backends.dask.execution.maps import *  # noqa: F403
+from ibis.backends.dask.execution.numeric import *  # noqa: F403
+from ibis.backends.dask.execution.reductions import *  # noqa: F403
+from ibis.backends.dask.execution.selection import *  # noqa: F403
+from ibis.backends.dask.execution.strings import *  # noqa: F403
+from ibis.backends.dask.execution.structs import *  # noqa: F403
+from ibis.backends.dask.execution.temporal import *  # noqa: F403
+from ibis.backends.dask.execution.window import *  # noqa: F403

ibis/backends/dask/execution/maps.py

@@ -2,7 +2,7 @@
 
 import dask.dataframe as dd
 import numpy as np
-import pandas
+import pandas as pd
 
 import ibis.expr.operations as ops
 from ibis.backends.dask.dispatch import execute_node
@@ -33,8 +33,8 @@
 # https://multiple-dispatch.readthedocs.io/en/latest/resolution.html#ambiguities
 # for more detail.
 PANDAS_REGISTERED_TYPES = [
-    (ops.MapGet, Mapping, object, pandas.Series),
-    (ops.MapGet, Mapping, pandas.Series, object),
+    (ops.MapGet, Mapping, object, pd.Series),
+    (ops.MapGet, Mapping, pd.Series, object),
 ]
 for registered_type in PANDAS_REGISTERED_TYPES:
     del execute_node[registered_type]
@@ -46,24 +46,15 @@
         ((dd.Series, object, object), map_get_series_scalar_scalar),
         ((dd.Series, object, dd.Series), map_get_series_scalar_series),
         ((dd.Series, dd.Series, object), map_get_series_series_scalar),
-        (
-            (dd.Series, dd.Series, dd.Series),
-            map_get_series_series_series,
-        ),
+        ((dd.Series, dd.Series, dd.Series), map_get_series_series_series),
         # This never occurs but we need to register it so multipledispatch
         # does not see below registrations as ambigious. See NOTE above.
         (
-            (Mapping, (dd.Series, pandas.Series), (dd.Series, pandas.Series)),
+            (Mapping, (dd.Series, pd.Series), (dd.Series, pd.Series)),
             map_get_dict_series_series,
         ),
-        (
-            (Mapping, object, (dd.Series, pandas.Series)),
-            map_get_dict_scalar_series,
-        ),
-        (
-            (Mapping, (dd.Series, pandas.Series), object),
-            map_get_dict_series_scalar,
-        ),
+        ((Mapping, object, (dd.Series, pd.Series)), map_get_dict_scalar_series),
+        ((Mapping, (dd.Series, pd.Series), object), map_get_dict_series_scalar),
     ],
     ops.MapContains: [
         ((Mapping, dd.Series), map_contains_dict_series),

ibis/backends/dask/execution/selection.py

@@ -7,7 +7,7 @@
 from typing import TYPE_CHECKING
 
 import dask.dataframe as dd
-import pandas
+import pandas as pd
 from toolz import concatv
 
 import ibis.expr.analysis as an
@@ -223,7 +223,7 @@ def execute_selection_dataframe(
         return result
 
     # create a sequence of columns that we need to drop
-    temporary_columns = pandas.Index(concatv(grouping_keys, ordering_keys)).difference(
+    temporary_columns = pd.Index(concatv(grouping_keys, ordering_keys)).difference(
         data.columns
     )
 

ibis/backends/dask/execution/strings.py

@@ -5,7 +5,7 @@
 import dask.dataframe as dd
 import dask.dataframe.groupby as ddgb
 import numpy as np
-import pandas
+import pandas as pd
 import toolz
 from pandas import isnull
 
@@ -254,7 +254,7 @@ def method(series, sep=sep):
 
 @execute_node.register(ops.StringAscii, dd.Series)
 def execute_string_ascii(op, data, **kwargs):
-    output_meta = pandas.Series([], dtype=np.dtype('int32'), name=data.name)
+    output_meta = pd.Series([], dtype=np.dtype('int32'), name=data.name)
     return data.map(ord, meta=output_meta)
 
 

ibis/backends/dask/execution/util.py

@@ -186,7 +186,7 @@ def safe_concat(dfs: list[dd.Series | dd.DataFrame]) -> dd.DataFrame:
     path is aggregations where aggregations return different numbers of rows
     (see `test_aggregation_group_by` for a specific example).
     TODO - performance.
-    """  # noqa: E501
+    """
     if len(dfs) == 1:
         maybe_df = dfs[0]
         if isinstance(maybe_df, dd.Series):

ibis/backends/dask/udf.py

@@ -5,7 +5,7 @@
 import dask.dataframe.groupby as ddgb
 import dask.delayed
 import numpy as np
-import pandas
+import pandas as pd
 
 import ibis.expr.operations as ops
 import ibis.expr.types as ir
@@ -78,7 +78,7 @@ def execute_udf_node(op, *args, cache=None, timecontext=None, **kwargs):
             df = dd.map_partitions(op.func, *args, meta=meta)
         else:
             name = args[0].name if len(args) == 1 else None
-            meta = pandas.Series([], name=name, dtype=op.return_type.to_dask())
+            meta = pd.Series([], name=name, dtype=op.return_type.to_dask())
             df = dd.map_partitions(op.func, *args, meta=meta)
 
         cache[(op, timecontext)] = df
@@ -108,7 +108,7 @@ def execute_udaf_node_no_groupby(op, *args, aggcontext, **kwargs):
         # passing that (now) pd.Series to aggctx. This materialization
         # happens at `.compute()` time, making this "lazy"
         @dask.delayed
-        def lazy_agg(*series: pandas.Series):
+        def lazy_agg(*series: pd.Series):
             return aggcontext.agg(series[0], op.func, *series[1:])
 
         lazy_result = lazy_agg(*args)
@@ -193,19 +193,19 @@ def apply_wrapper(df, apply_func, col_names):
             return apply_func(*cols)
 
         if len(groupings) > 1:
-            meta_index = pandas.MultiIndex.from_arrays(
+            meta_index = pd.MultiIndex.from_arrays(
                 [[0]] * len(groupings), names=groupings
             )
             meta_value = [dd.utils.make_meta(out_type)]
         else:
-            meta_index = pandas.Index([], name=groupings[0])
+            meta_index = pd.Index([], name=groupings[0])
             meta_value = []
 
         return grouped_df.apply(
             apply_wrapper,
             func,
             col_names,
-            meta=pandas.Series(meta_value, index=meta_index, dtype=out_type),
+            meta=pd.Series(meta_value, index=meta_index, dtype=out_type),
         )
 
     @execute_node.register(
@@ -254,7 +254,7 @@ def apply_wrapper(df, apply_func, col_names):
         else:
             # after application we will get a series with a multi-index of
             # groupings + index
-            meta_index = pandas.MultiIndex.from_arrays(
+            meta_index = pd.MultiIndex.from_arrays(
                 [[0]] * (len(groupings) + 1),
                 names=groupings + [parent_df.index.name],
             )
@@ -264,7 +264,7 @@ def apply_wrapper(df, apply_func, col_names):
                 apply_wrapper,
                 func,
                 col_names,
-                meta=pandas.Series(meta_value, index=meta_index, dtype=out_type),
+                meta=pd.Series(meta_value, index=meta_index, dtype=out_type),
             )
 
         return result