diff --git a/docs/backends/impala.qmd b/docs/backends/impala.qmd index c6e4b80f1acf..e5512c384c3f 100644 --- a/docs/backends/impala.qmd +++ b/docs/backends/impala.qmd @@ -202,7 +202,7 @@ table or database. ```{python} #| echo: false #| output: asis -render_methods(get_object("ibis.backends.base.sqlglot", "SQLGlotBackend"), "table") +render_methods(get_object("ibis.backends.sql", "SQLBackend"), "table") ``` The client's `table` method allows you to create an Ibis table diff --git a/docs/support_matrix.py b/docs/support_matrix.py index 94e658ebccfb..0fb11337dac9 100644 --- a/docs/support_matrix.py +++ b/docs/support_matrix.py @@ -9,7 +9,7 @@ def make_support_matrix(): """Construct the backend operation support matrix data.""" - from ibis.backends.base.sqlglot.compiler import ALL_OPERATIONS + from ibis.backends.sql.compiler import ALL_OPERATIONS support_matrix_ignored_operations = (ops.ScalarParameter,) diff --git a/docs/support_matrix.qmd b/docs/support_matrix.qmd index eb0087e29201..73b8b45311e2 100644 --- a/docs/support_matrix.qmd +++ b/docs/support_matrix.qmd @@ -44,12 +44,11 @@ dict( #| content: valuebox #| title: "Number of SQL backends" import importlib -from ibis.backends.base.sqlglot import SQLGlotBackend +from ibis.backends.sql import SQLBackend sql_backends = sum( issubclass( - importlib.import_module(f"ibis.backends.{entry_point.name}").Backend, - SQLGlotBackend + importlib.import_module(f"ibis.backends.{entry_point.name}").Backend, SQLBackend ) for entry_point in ibis.util.backend_entry_points() ) diff --git a/ibis/__init__.py b/ibis/__init__.py index 8688490a66bd..856f522a6cbb 100644 --- a/ibis/__init__.py +++ b/ibis/__init__.py @@ -4,7 +4,7 @@ __version__ = "8.0.0" from ibis import examples, util -from ibis.backends.base import BaseBackend +from ibis.backends import BaseBackend from ibis.common.exceptions import IbisError from ibis.config import options from ibis.expr import api diff --git a/ibis/backends/__init__.py b/ibis/backends/__init__.py index e69de29bb2d1..378d82252c95 100644 --- a/ibis/backends/__init__.py +++ b/ibis/backends/__init__.py @@ -0,0 +1,1459 @@ +from __future__ import annotations + +import abc +import collections.abc +import functools +import importlib.metadata +import keyword +import re +import sys +import urllib.parse +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, ClassVar +from urllib.parse import parse_qs, urlparse + +import ibis +import ibis.common.exceptions as exc +import ibis.config +import ibis.expr.operations as ops +import ibis.expr.types as ir +from ibis import util +from ibis.common.caching import RefCountedCache + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, Mapping, MutableMapping + + import pandas as pd + import pyarrow as pa + import sqlglot as sg + import torch + +__all__ = ("BaseBackend", "Database", "connect") + + +class Database: + """Generic Database class.""" + + def __init__(self, name: str, client: Any) -> None: + self.name = name + self.client = client + + def __repr__(self) -> str: + """Return type name and the name of the database.""" + return f"{type(self).__name__}({self.name!r})" + + def __dir__(self) -> list[str]: + """Return the attributes and tables of the database. + + Returns + ------- + list[str] + A list of the attributes and tables available in the database. + + """ + attrs = dir(type(self)) + unqualified_tables = [self._unqualify(x) for x in self.tables] + return sorted(frozenset(attrs + unqualified_tables)) + + def __contains__(self, table: str) -> bool: + """Check if the given table is available in the current database. + + Parameters + ---------- + table + Table name + + Returns + ------- + bool + True if the given table is available in the current database. + + """ + return table in self.tables + + @property + def tables(self) -> list[str]: + """Return a list with all available tables. + + Returns + ------- + list[str] + The list of tables in the database + + """ + return self.list_tables() + + def __getitem__(self, table: str) -> ir.Table: + """Return a Table for the given table name. + + Parameters + ---------- + table + Table name + + Returns + ------- + Table + Table expression + + """ + return self.table(table) + + def __getattr__(self, table: str) -> ir.Table: + """Return a Table for the given table name. + + Parameters + ---------- + table + Table name + + Returns + ------- + Table + Table expression + + """ + return self.table(table) + + def _qualify(self, value): + return value + + def _unqualify(self, value): + return value + + def drop(self, force: bool = False) -> None: + """Drop the database. + + Parameters + ---------- + force + If `True`, drop any objects that exist, and do not fail if the + database does not exist. + + """ + self.client.drop_database(self.name, force=force) + + def table(self, name: str) -> ir.Table: + """Return a table expression referencing a table in this database. + + Parameters + ---------- + name + The name of a table + + Returns + ------- + Table + Table expression + + """ + qualified_name = self._qualify(name) + return self.client.table(qualified_name, self.name) + + def list_tables(self, like=None, database=None): + """List the tables in the database. + + Parameters + ---------- + like + A pattern to use for listing tables. + database + The database to perform the list against + + """ + return self.client.list_tables(like, database=database or self.name) + + +class TablesAccessor(collections.abc.Mapping): + """A mapping-like object for accessing tables off a backend. + + Tables may be accessed by name using either index or attribute access: + + Examples + -------- + >>> con = ibis.sqlite.connect("example.db") + >>> people = con.tables["people"] # access via index + >>> people = con.tables.people # access via attribute + + """ + + def __init__(self, backend: BaseBackend): + self._backend = backend + + def __getitem__(self, name) -> ir.Table: + try: + return self._backend.table(name) + except Exception as exc: # noqa: BLE001 + raise KeyError(name) from exc + + def __getattr__(self, name) -> ir.Table: + if name.startswith("_"): + raise AttributeError(name) + try: + return self._backend.table(name) + except Exception as exc: # noqa: BLE001 + raise AttributeError(name) from exc + + def __iter__(self) -> Iterator[str]: + return iter(sorted(self._backend.list_tables())) + + def __len__(self) -> int: + return len(self._backend.list_tables()) + + def __dir__(self) -> list[str]: + o = set() + o.update(dir(type(self))) + o.update( + name + for name in self._backend.list_tables() + if name.isidentifier() and not keyword.iskeyword(name) + ) + return list(o) + + def __repr__(self) -> str: + tables = self._backend.list_tables() + rows = ["Tables", "------"] + rows.extend(f"- {name}" for name in sorted(tables)) + return "\n".join(rows) + + def _ipython_key_completions_(self) -> list[str]: + return self._backend.list_tables() + + +class _FileIOHandler: + @staticmethod + def _import_pyarrow(): + try: + import pyarrow # noqa: ICN001 + except ImportError: + raise ModuleNotFoundError( + "Exporting to arrow formats requires `pyarrow` but it is not installed" + ) + else: + import pyarrow_hotfix # noqa: F401 + + return pyarrow + + def to_pandas( + self, + expr: ir.Expr, + *, + params: Mapping[ir.Scalar, Any] | None = None, + limit: int | str | None = None, + **kwargs: Any, + ) -> pd.DataFrame | pd.Series | Any: + """Execute an Ibis expression and return a pandas `DataFrame`, `Series`, or scalar. + + ::: {.callout-note} + This method is a wrapper around `execute`. + ::: + + Parameters + ---------- + expr + Ibis expression to execute. + params + Mapping of scalar parameter expressions to value. + limit + An integer to effect a specific row limit. A value of `None` means + "no limit". The default is in `ibis/config.py`. + kwargs + Keyword arguments + + """ + return self.execute(expr, params=params, limit=limit, **kwargs) + + def to_pandas_batches( + self, + expr: ir.Expr, + *, + params: Mapping[ir.Scalar, Any] | None = None, + limit: int | str | None = None, + chunk_size: int = 1_000_000, + **kwargs: Any, + ) -> Iterator[pd.DataFrame | pd.Series | Any]: + """Execute an Ibis expression and return an iterator of pandas `DataFrame`s. + + Parameters + ---------- + expr + Ibis expression to execute. + params + Mapping of scalar parameter expressions to value. + limit + An integer to effect a specific row limit. A value of `None` means + "no limit". The default is in `ibis/config.py`. + chunk_size + Maximum number of rows in each returned `DataFrame` batch. This may have + no effect depending on the backend. + kwargs + Keyword arguments + + Returns + ------- + Iterator[pd.DataFrame] + An iterator of pandas `DataFrame`s. + + """ + from ibis.formats.pandas import PandasData + + orig_expr = expr + expr = expr.as_table() + schema = expr.schema() + yield from ( + orig_expr.__pandas_result__( + PandasData.convert_table(batch.to_pandas(), schema) + ) + for batch in self.to_pyarrow_batches( + expr, params=params, limit=limit, chunk_size=chunk_size, **kwargs + ) + ) + + @util.experimental + def to_pyarrow( + self, + expr: ir.Expr, + *, + params: Mapping[ir.Scalar, Any] | None = None, + limit: int | str | None = None, + **kwargs: Any, + ) -> pa.Table: + """Execute expression and return results in as a pyarrow table. + + This method is eager and will execute the associated expression + immediately. + + Parameters + ---------- + expr + Ibis expression to export to pyarrow + params + Mapping of scalar parameter expressions to value. + limit + An integer to effect a specific row limit. A value of `None` means + "no limit". The default is in `ibis/config.py`. + kwargs + Keyword arguments + + Returns + ------- + Table + A pyarrow table holding the results of the executed expression. + + """ + pa = self._import_pyarrow() + self._run_pre_execute_hooks(expr) + + table_expr = expr.as_table() + schema = table_expr.schema() + arrow_schema = schema.to_pyarrow() + with self.to_pyarrow_batches( + table_expr, params=params, limit=limit, **kwargs + ) as reader: + table = pa.Table.from_batches(reader, schema=arrow_schema) + + return expr.__pyarrow_result__( + table.rename_columns(table_expr.columns).cast(arrow_schema) + ) + + @util.experimental + def to_pyarrow_batches( + self, + expr: ir.Expr, + *, + params: Mapping[ir.Scalar, Any] | None = None, + limit: int | str | None = None, + chunk_size: int = 1_000_000, + **kwargs: Any, + ) -> pa.ipc.RecordBatchReader: + """Execute expression and return a RecordBatchReader. + + This method is eager and will execute the associated expression + immediately. + + Parameters + ---------- + expr + Ibis expression to export to pyarrow + limit + An integer to effect a specific row limit. A value of `None` means + "no limit". The default is in `ibis/config.py`. + params + Mapping of scalar parameter expressions to value. + chunk_size + Maximum number of rows in each returned record batch. + kwargs + Keyword arguments + + Returns + ------- + results + RecordBatchReader + + """ + raise NotImplementedError + + @util.experimental + def to_torch( + self, + expr: ir.Expr, + *, + params: Mapping[ir.Scalar, Any] | None = None, + limit: int | str | None = None, + **kwargs: Any, + ) -> dict[str, torch.Tensor]: + """Execute an expression and return results as a dictionary of torch tensors. + + Parameters + ---------- + expr + Ibis expression to execute. + params + Parameters to substitute into the expression. + limit + An integer to effect a specific row limit. A value of `None` means no limit. + kwargs + Keyword arguments passed into the backend's `to_torch` implementation. + + Returns + ------- + dict[str, torch.Tensor] + A dictionary of torch tensors, keyed by column name. + + """ + import torch + + t = self.to_pyarrow(expr, params=params, limit=limit, **kwargs) + # without .copy() the arrays are read-only and thus writing to them is + # undefined behavior; we can't ignore this warning from torch because + # we're going out of ibis and downstream code can do whatever it wants + # with the data + return { + name: torch.from_numpy(t[name].to_numpy().copy()) for name in t.schema.names + } + + def read_parquet( + self, path: str | Path, table_name: str | None = None, **kwargs: Any + ) -> ir.Table: + """Register a parquet file as a table in the current backend. + + Parameters + ---------- + path + The data source. + table_name + An optional name to use for the created table. This defaults to + a sequentially generated name. + **kwargs + Additional keyword arguments passed to the backend loading function. + + Returns + ------- + ir.Table + The just-registered table + + """ + raise NotImplementedError( + f"{self.name} does not support direct registration of parquet data." + ) + + def read_csv( + self, path: str | Path, table_name: str | None = None, **kwargs: Any + ) -> ir.Table: + """Register a CSV file as a table in the current backend. + + Parameters + ---------- + path + The data source. A string or Path to the CSV file. + table_name + An optional name to use for the created table. This defaults to + a sequentially generated name. + **kwargs + Additional keyword arguments passed to the backend loading function. + + Returns + ------- + ir.Table + The just-registered table + + """ + raise NotImplementedError( + f"{self.name} does not support direct registration of CSV data." + ) + + def read_json( + self, path: str | Path, table_name: str | None = None, **kwargs: Any + ) -> ir.Table: + """Register a JSON file as a table in the current backend. + + Parameters + ---------- + path + The data source. A string or Path to the JSON file. + table_name + An optional name to use for the created table. This defaults to + a sequentially generated name. + **kwargs + Additional keyword arguments passed to the backend loading function. + + Returns + ------- + ir.Table + The just-registered table + + """ + raise NotImplementedError( + f"{self.name} does not support direct registration of JSON data." + ) + + def read_delta( + self, source: str | Path, table_name: str | None = None, **kwargs: Any + ): + """Register a Delta Lake table in the current database. + + Parameters + ---------- + source + The data source. Must be a directory + containing a Delta Lake table. + table_name + An optional name to use for the created table. This defaults to + a sequentially generated name. + **kwargs + Additional keyword arguments passed to the underlying backend or library. + + Returns + ------- + ir.Table + The just-registered table. + + """ + raise NotImplementedError( + f"{self.name} does not support direct registration of DeltaLake tables." + ) + + @util.experimental + def to_parquet( + self, + expr: ir.Table, + path: str | Path, + *, + params: Mapping[ir.Scalar, Any] | None = None, + **kwargs: Any, + ) -> None: + """Write the results of executing the given expression to a parquet file. + + This method is eager and will execute the associated expression + immediately. + + Parameters + ---------- + expr + The ibis expression to execute and persist to parquet. + path + The data source. A string or Path to the parquet file. + params + Mapping of scalar parameter expressions to value. + **kwargs + Additional keyword arguments passed to pyarrow.parquet.ParquetWriter + + https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html + + """ + self._import_pyarrow() + import pyarrow.parquet as pq + + with expr.to_pyarrow_batches(params=params) as batch_reader: + with pq.ParquetWriter(path, batch_reader.schema, **kwargs) as writer: + for batch in batch_reader: + writer.write_batch(batch) + + @util.experimental + def to_csv( + self, + expr: ir.Table, + path: str | Path, + *, + params: Mapping[ir.Scalar, Any] | None = None, + **kwargs: Any, + ) -> None: + """Write the results of executing the given expression to a CSV file. + + This method is eager and will execute the associated expression + immediately. + + Parameters + ---------- + expr + The ibis expression to execute and persist to CSV. + path + The data source. A string or Path to the CSV file. + params + Mapping of scalar parameter expressions to value. + kwargs + Additional keyword arguments passed to pyarrow.csv.CSVWriter + + https://arrow.apache.org/docs/python/generated/pyarrow.csv.CSVWriter.html + + """ + self._import_pyarrow() + import pyarrow.csv as pcsv + + with expr.to_pyarrow_batches(params=params) as batch_reader: + with pcsv.CSVWriter(path, batch_reader.schema, **kwargs) as writer: + for batch in batch_reader: + writer.write_batch(batch) + + @util.experimental + def to_delta( + self, + expr: ir.Table, + path: str | Path, + *, + params: Mapping[ir.Scalar, Any] | None = None, + **kwargs: Any, + ) -> None: + """Write the results of executing the given expression to a Delta Lake table. + + This method is eager and will execute the associated expression + immediately. + + Parameters + ---------- + expr + The ibis expression to execute and persist to Delta Lake table. + path + The data source. A string or Path to the Delta Lake table. + params + Mapping of scalar parameter expressions to value. + kwargs + Additional keyword arguments passed to deltalake.writer.write_deltalake method + + """ + try: + from deltalake.writer import write_deltalake + except ImportError: + raise ImportError( + "The deltalake extra is required to use the " + "to_delta method. You can install it using pip:\n\n" + "pip install 'ibis-framework[deltalake]'\n" + ) + + with expr.to_pyarrow_batches(params=params) as batch_reader: + write_deltalake(path, batch_reader, **kwargs) + + +class CanListDatabases(abc.ABC): + @abc.abstractmethod + def list_databases(self, like: str | None = None) -> list[str]: + """List existing databases in the current connection. + + Parameters + ---------- + like + A pattern in Python's regex format to filter returned database + names. + + Returns + ------- + list[str] + The database names that exist in the current connection, that match + the `like` pattern if provided. + + """ + + @property + @abc.abstractmethod + def current_database(self) -> str: + """The current database in use.""" + + +class CanCreateDatabase(CanListDatabases): + @abc.abstractmethod + def create_database(self, name: str, force: bool = False) -> None: + """Create a new database. + + Parameters + ---------- + name + Name of the new database. + force + If `False`, an exception is raised if the database already exists. + + """ + + @abc.abstractmethod + def drop_database(self, name: str, force: bool = False) -> None: + """Drop a database with name `name`. + + Parameters + ---------- + name + Database to drop. + force + If `False`, an exception is raised if the database does not exist. + + """ + + +class CanCreateSchema(abc.ABC): + @abc.abstractmethod + def create_schema( + self, name: str, database: str | None = None, force: bool = False + ) -> None: + """Create a schema named `name` in `database`. + + Parameters + ---------- + name + Name of the schema to create. + database + Name of the database in which to create the schema. If `None`, the + current database is used. + force + If `False`, an exception is raised if the schema exists. + + """ + + @abc.abstractmethod + def drop_schema( + self, name: str, database: str | None = None, force: bool = False + ) -> None: + """Drop the schema with `name` in `database`. + + Parameters + ---------- + name + Name of the schema to drop. + database + Name of the database to drop the schema from. If `None`, the + current database is used. + force + If `False`, an exception is raised if the schema does not exist. + + """ + + @abc.abstractmethod + def list_schemas( + self, like: str | None = None, database: str | None = None + ) -> list[str]: + """List existing schemas in the current connection. + + Parameters + ---------- + like + A pattern in Python's regex format to filter returned schema + names. + database + The database to list schemas from. If `None`, the current database + is searched. + + Returns + ------- + list[str] + The schema names that exist in the current connection, that match + the `like` pattern if provided. + + """ + + @property + @abc.abstractmethod + def current_schema(self) -> str: + """Return the current schema.""" + + +class BaseBackend(abc.ABC, _FileIOHandler): + """Base backend class. + + All Ibis backends must subclass this class and implement all the + required methods. + """ + + name: ClassVar[str] + + supports_temporary_tables = False + supports_python_udfs = False + supports_in_memory_tables = True + + def __init__(self, *args, **kwargs): + self._con_args: tuple[Any] = args + self._con_kwargs: dict[str, Any] = kwargs + # expression cache + self._query_cache = RefCountedCache( + populate=self._load_into_cache, + lookup=lambda name: self.table(name).op(), + finalize=self._clean_up_cached_table, + generate_name=functools.partial(util.gen_name, "cache"), + key=lambda expr: expr.op(), + ) + + @property + @abc.abstractmethod + def dialect(self) -> sg.Dialect | None: + """The sqlglot dialect for this backend, where applicable. + + Returns None if the backend is not a SQL backend. + """ + + def __getstate__(self): + return dict(_con_args=self._con_args, _con_kwargs=self._con_kwargs) + + def __rich_repr__(self): + yield "name", self.name + + def __hash__(self): + return hash(self.db_identity) + + def __eq__(self, other): + return self.db_identity == other.db_identity + + @functools.cached_property + def db_identity(self) -> str: + """Return the identity of the database. + + Multiple connections to the same + database will return the same value for `db_identity`. + + The default implementation assumes connection parameters uniquely + specify the database. + + Returns + ------- + Hashable + Database identity + + """ + parts = [self.__class__] + parts.extend(self._con_args) + parts.extend(f"{k}={v}" for k, v in self._con_kwargs.items()) + return "_".join(map(str, parts)) + + # TODO(kszucs): this should be a classmethod returning with a new backend + # instance which does instantiate the connection + def connect(self, *args, **kwargs) -> BaseBackend: + """Connect to the database. + + Parameters + ---------- + *args + Mandatory connection parameters, see the docstring of `do_connect` + for details. + **kwargs + Extra connection parameters, see the docstring of `do_connect` for + details. + + Notes + ----- + This creates a new backend instance with saved `args` and `kwargs`, + then calls `reconnect` and finally returns the newly created and + connected backend instance. + + Returns + ------- + BaseBackend + An instance of the backend + + """ + new_backend = self.__class__(*args, **kwargs) + new_backend.reconnect() + return new_backend + + @abc.abstractmethod + def disconnect(self) -> None: + """Close the connection to the backend.""" + + @staticmethod + def _convert_kwargs(kwargs: MutableMapping) -> None: + """Manipulate keyword arguments to `.connect` method.""" + + # TODO(kszucs): should call self.connect(*self._con_args, **self._con_kwargs) + def reconnect(self) -> None: + """Reconnect to the database already configured with connect.""" + self.do_connect(*self._con_args, **self._con_kwargs) + + def do_connect(self, *args, **kwargs) -> None: + """Connect to database specified by `args` and `kwargs`.""" + + @util.deprecated(instead="use equivalent methods in the backend") + def database(self, name: str | None = None) -> Database: + """Return a `Database` object for the `name` database. + + Parameters + ---------- + name + Name of the database to return the object for. + + Returns + ------- + Database + A database object for the specified database. + + """ + return Database(name=name or self.current_database, client=self) + + @staticmethod + def _filter_with_like(values: Iterable[str], like: str | None = None) -> list[str]: + """Filter names with a `like` pattern (regex). + + The methods `list_databases` and `list_tables` accept a `like` + argument, which filters the returned tables with tables that match the + provided pattern. + + We provide this method in the base backend, so backends can use it + instead of reinventing the wheel. + + Parameters + ---------- + values + Iterable of strings to filter + like + Pattern to use for filtering names + + Returns + ------- + list[str] + Names filtered by the `like` pattern. + + """ + if like is None: + return sorted(values) + + pattern = re.compile(like) + return sorted(filter(pattern.findall, values)) + + @abc.abstractmethod + def list_tables( + self, like: str | None = None, database: str | None = None + ) -> list[str]: + """Return the list of table names in the current database. + + For some backends, the tables may be files in a directory, + or other equivalent entities in a SQL database. + + Parameters + ---------- + like + A pattern in Python's regex format. + database + The database from which to list tables. If not provided, the + current database is used. + + Returns + ------- + list[str] + The list of the table names that match the pattern `like`. + + """ + + @abc.abstractmethod + def table(self, name: str, database: str | None = None) -> ir.Table: + """Construct a table expression. + + Parameters + ---------- + name + Table name + database + Database name + + Returns + ------- + Table + Table expression + + """ + + @functools.cached_property + def tables(self): + """An accessor for tables in the database. + + Tables may be accessed by name using either index or attribute access: + + Examples + -------- + >>> con = ibis.sqlite.connect("example.db") + >>> people = con.tables["people"] # access via index + >>> people = con.tables.people # access via attribute + + """ + return TablesAccessor(self) + + @property + @abc.abstractmethod + def version(self) -> str: + """Return the version of the backend engine. + + For database servers, return the server version. + + For others such as SQLite and pandas return the version of the + underlying library or application. + + Returns + ------- + str + The backend version + + """ + + @classmethod + def register_options(cls) -> None: + """Register custom backend options.""" + options = ibis.config.options + backend_name = cls.name + try: + backend_options = cls.Options() + except AttributeError: + pass + else: + try: + setattr(options, backend_name, backend_options) + except ValueError as e: + raise exc.BackendConfigurationNotRegistered(backend_name) from e + + def _register_udfs(self, expr: ir.Expr) -> None: + """Register UDFs contained in `expr` with the backend.""" + if self.supports_python_udfs: + raise NotImplementedError(self.name) + + def _register_in_memory_tables(self, expr: ir.Expr): + if self.supports_in_memory_tables: + raise NotImplementedError(self.name) + + def _run_pre_execute_hooks(self, expr: ir.Expr) -> None: + """Backend-specific hooks to run before an expression is executed.""" + self._define_udf_translation_rules(expr) + self._register_udfs(expr) + self._register_in_memory_tables(expr) + + def _define_udf_translation_rules(self, expr: ir.Expr): + if self.supports_python_udfs: + raise NotImplementedError(self.name) + + def compile( + self, + expr: ir.Expr, + params: Mapping[ir.Expr, Any] | None = None, + ) -> Any: + """Compile an expression.""" + return self.compiler.to_sql(expr, params=params) + + def _to_sql(self, expr: ir.Expr, **kwargs) -> str: + """Convert an expression to a SQL string. + + Called by `ibis.to_sql`; gives the backend an opportunity to generate + nicer SQL for human consumption. + """ + raise NotImplementedError(f"Backend '{self.name}' backend doesn't support SQL") + + def execute(self, expr: ir.Expr) -> Any: + """Execute an expression.""" + + def add_operation(self, operation: ops.Node) -> Callable: + """Add a translation function to the backend for a specific operation. + + Operations are defined in `ibis.expr.operations`, and a translation + function receives the translator object and an expression as + parameters, and returns a value depending on the backend. + """ + if not hasattr(self, "compiler"): + raise RuntimeError("Only SQL-based backends support `add_operation`") + + def decorator(translation_function: Callable) -> None: + self.compiler.translator_class.add_operation( + operation, translation_function + ) + + return decorator + + @abc.abstractmethod + def create_table( + self, + name: str, + obj: pd.DataFrame | pa.Table | ir.Table | None = None, + *, + schema: ibis.Schema | None = None, + database: str | None = None, + temp: bool = False, + overwrite: bool = False, + ) -> ir.Table: + """Create a new table. + + Parameters + ---------- + name + Name of the new table. + obj + An Ibis table expression or pandas table that will be used to + extract the schema and the data of the new table. If not provided, + `schema` must be given. + schema + The schema for the new table. Only one of `schema` or `obj` can be + provided. + database + Name of the database where the table will be created, if not the + default. + temp + Whether a table is temporary or not + overwrite + Whether to clobber existing data + + Returns + ------- + Table + The table that was created. + + """ + + @abc.abstractmethod + def drop_table( + self, + name: str, + *, + database: str | None = None, + force: bool = False, + ) -> None: + """Drop a table. + + Parameters + ---------- + name + Name of the table to drop. + database + Name of the database where the table exists, if not the default. + force + If `False`, an exception is raised if the table does not exist. + + """ + raise NotImplementedError( + f'Backend "{self.name}" does not implement "drop_table"' + ) + + def rename_table(self, old_name: str, new_name: str) -> None: + """Rename an existing table. + + Parameters + ---------- + old_name + The old name of the table. + new_name + The new name of the table. + + """ + raise NotImplementedError( + f'Backend "{self.name}" does not implement "rename_table"' + ) + + @abc.abstractmethod + def create_view( + self, + name: str, + obj: ir.Table, + *, + database: str | None = None, + overwrite: bool = False, + ) -> ir.Table: + """Create a new view from an expression. + + Parameters + ---------- + name + Name of the new view. + obj + An Ibis table expression that will be used to create the view. + database + Name of the database where the view will be created, if not + provided the database's default is used. + overwrite + Whether to clobber an existing view with the same name + + Returns + ------- + Table + The view that was created. + + """ + + @abc.abstractmethod + def drop_view( + self, name: str, *, database: str | None = None, force: bool = False + ) -> None: + """Drop a view. + + Parameters + ---------- + name + Name of the view to drop. + database + Name of the database where the view exists, if not the default. + force + If `False`, an exception is raised if the view does not exist. + + """ + + @classmethod + def has_operation(cls, operation: type[ops.Value]) -> bool: + """Return whether the backend implements support for `operation`. + + Parameters + ---------- + operation + A class corresponding to an operation. + + Returns + ------- + bool + Whether the backend implements the operation. + + Examples + -------- + >>> import ibis + >>> import ibis.expr.operations as ops + >>> ibis.sqlite.has_operation(ops.ArrayIndex) + False + >>> ibis.postgres.has_operation(ops.ArrayIndex) + True + + """ + raise NotImplementedError( + f"{cls.name} backend has not implemented `has_operation` API" + ) + + def _cached(self, expr: ir.Table): + """Cache the provided expression. + + All subsequent operations on the returned expression will be performed on the cached data. + + Parameters + ---------- + expr + Table expression to cache + + Returns + ------- + Expr + Cached table + + """ + op = expr.op() + if (result := self._query_cache.get(op)) is None: + self._query_cache.store(expr) + result = self._query_cache[op] + return ir.CachedTable(result) + + def _release_cached(self, expr: ir.CachedTable) -> None: + """Releases the provided cached expression. + + Parameters + ---------- + expr + Cached expression to release + + """ + del self._query_cache[expr.op()] + + def _load_into_cache(self, name, expr): + raise NotImplementedError(self.name) + + def _clean_up_cached_table(self, op): + raise NotImplementedError(self.name) + + def _transpile_sql(self, query: str, *, dialect: str | None = None) -> str: + # only transpile if dialect was passed + if dialect is None: + return query + + import sqlglot as sg + + # only transpile if the backend dialect doesn't match the input dialect + name = self.name + if (output_dialect := self.dialect) is None: + raise NotImplementedError(f"No known sqlglot dialect for backend {name}") + + if dialect != output_dialect: + (query,) = sg.transpile(query, read=dialect, write=output_dialect) + return query + + +@functools.cache +def _get_backend_names(*, exclude: tuple[str] = ()) -> frozenset[str]: + """Return the set of known backend names. + + Parameters + ---------- + exclude + Exclude these backend names from the result + + Notes + ----- + This function returns a frozenset to prevent cache pollution. + + If a `set` is used, then any in-place modifications to the set + are visible to every caller of this function. + + """ + + if sys.version_info < (3, 10): + entrypoints = importlib.metadata.entry_points()["ibis.backends"] + else: + entrypoints = importlib.metadata.entry_points(group="ibis.backends") + return frozenset(ep.name for ep in entrypoints).difference(exclude) + + +def connect(resource: Path | str, **kwargs: Any) -> BaseBackend: + """Connect to `resource`, inferring the backend automatically. + + The general pattern for `ibis.connect` is + + ```python + con = ibis.connect("backend://connection-parameters") + ``` + + With many backends that looks like + + ```python + con = ibis.connect("backend://user:password@host:port/database") + ``` + + See the connection syntax for each backend for details about URL connection + requirements. + + Parameters + ---------- + resource + A URL or path to the resource to be connected to. + kwargs + Backend specific keyword arguments + + Examples + -------- + Connect to an in-memory DuckDB database: + + >>> import ibis + >>> con = ibis.connect("duckdb://") + + Connect to an on-disk SQLite database: + + >>> con = ibis.connect("sqlite://relative.db") + >>> con = ibis.connect( + ... "sqlite:///absolute/path/to/data.db" + ... ) # quartodoc: +SKIP # doctest: +SKIP + + Connect to a PostgreSQL server: + + >>> con = ibis.connect( + ... "postgres://user:password@hostname:5432" + ... ) # quartodoc: +SKIP # doctest: +SKIP + + Connect to BigQuery: + + >>> con = ibis.connect( + ... "bigquery://my-project/my-dataset" + ... ) # quartodoc: +SKIP # doctest: +SKIP + + """ + url = resource = str(resource) + + if re.match("[A-Za-z]:", url): + # windows path with drive, treat it as a file + url = f"file://{url}" + + parsed = urllib.parse.urlparse(url) + scheme = parsed.scheme or "file" + + orig_kwargs = kwargs.copy() + kwargs = dict(urllib.parse.parse_qsl(parsed.query)) + + if scheme == "file": + path = parsed.netloc + parsed.path + # Merge explicit kwargs with query string, explicit kwargs + # taking precedence + kwargs.update(orig_kwargs) + if path.endswith(".duckdb"): + return ibis.duckdb.connect(path, **kwargs) + elif path.endswith((".sqlite", ".db")): + return ibis.sqlite.connect(path, **kwargs) + elif path.endswith((".parquet", ".csv", ".csv.gz")): + # Load parquet/csv/csv.gz files with duckdb by default + con = ibis.duckdb.connect(**kwargs) + con.register(path) + return con + else: + raise ValueError(f"Don't know how to connect to {resource!r}") + + if kwargs: + # If there are kwargs (either explicit or from the query string), + # re-add them to the parsed URL + query = urllib.parse.urlencode(kwargs) + parsed = parsed._replace(query=query) + + if scheme in ("postgres", "postgresql"): + # Treat `postgres://` and `postgresql://` the same + scheme = "postgres" + + # Convert all arguments back to a single URL string + url = parsed.geturl() + if "://" not in url: + # urllib may roundtrip `duckdb://` to `duckdb:`. Here we re-add the + # missing `//`. + url = url.replace(":", "://", 1) + + try: + backend = getattr(ibis, scheme) + except AttributeError: + raise ValueError(f"Don't know how to connect to {resource!r}") from None + + return backend._from_url(url, **orig_kwargs) + + +class UrlFromPath: + __slots__ = () + + def _from_url(self, url: str, **kwargs) -> BaseBackend: + """Connect to a backend using a URL `url`. + + Parameters + ---------- + url + URL with which to connect to a backend. + kwargs + Additional keyword arguments + + Returns + ------- + BaseBackend + A backend instance + + """ + url = urlparse(url) + netloc = url.netloc + parts = list(filter(None, (netloc, url.path[bool(netloc) :]))) + database = Path(*parts) if parts and parts != [":memory:"] else ":memory:" + if (strdatabase := str(database)).startswith("md:") or strdatabase.startswith( + "motherduck:" + ): + database = strdatabase + elif isinstance(database, Path): + database = database.absolute() + + query_params = parse_qs(url.query) + + for name, value in query_params.items(): + if len(value) > 1: + kwargs[name] = value + elif len(value) == 1: + kwargs[name] = value[0] + else: + raise exc.IbisError(f"Invalid URL parameter: {name}") + + self._convert_kwargs(kwargs) + return self.connect(database=database, **kwargs) + + +class NoUrl: + __slots__ = () + + name: str + + def _from_url(self, url: str, **_) -> ir.Table: + raise NotImplementedError(self.name) diff --git a/ibis/backends/base/__init__.py b/ibis/backends/base/__init__.py deleted file mode 100644 index 58a20a40a9aa..000000000000 --- a/ibis/backends/base/__init__.py +++ /dev/null @@ -1,1464 +0,0 @@ -from __future__ import annotations - -import abc -import collections.abc -import functools -import importlib.metadata -import keyword -import re -import sys -import urllib.parse -from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Callable, - ClassVar, -) -from urllib.parse import parse_qs, urlparse - -import ibis -import ibis.common.exceptions as exc -import ibis.config -import ibis.expr.operations as ops -import ibis.expr.types as ir -from ibis import util -from ibis.common.caching import RefCountedCache - -if TYPE_CHECKING: - from collections.abc import Iterable, Iterator, Mapping, MutableMapping - - import pandas as pd - import pyarrow as pa - import sqlglot as sg - import torch - -__all__ = ("BaseBackend", "Database", "connect") - - -class Database: - """Generic Database class.""" - - def __init__(self, name: str, client: Any) -> None: - self.name = name - self.client = client - - def __repr__(self) -> str: - """Return type name and the name of the database.""" - return f"{type(self).__name__}({self.name!r})" - - def __dir__(self) -> list[str]: - """Return the attributes and tables of the database. - - Returns - ------- - list[str] - A list of the attributes and tables available in the database. - - """ - attrs = dir(type(self)) - unqualified_tables = [self._unqualify(x) for x in self.tables] - return sorted(frozenset(attrs + unqualified_tables)) - - def __contains__(self, table: str) -> bool: - """Check if the given table is available in the current database. - - Parameters - ---------- - table - Table name - - Returns - ------- - bool - True if the given table is available in the current database. - - """ - return table in self.tables - - @property - def tables(self) -> list[str]: - """Return a list with all available tables. - - Returns - ------- - list[str] - The list of tables in the database - - """ - return self.list_tables() - - def __getitem__(self, table: str) -> ir.Table: - """Return a Table for the given table name. - - Parameters - ---------- - table - Table name - - Returns - ------- - Table - Table expression - - """ - return self.table(table) - - def __getattr__(self, table: str) -> ir.Table: - """Return a Table for the given table name. - - Parameters - ---------- - table - Table name - - Returns - ------- - Table - Table expression - - """ - return self.table(table) - - def _qualify(self, value): - return value - - def _unqualify(self, value): - return value - - def drop(self, force: bool = False) -> None: - """Drop the database. - - Parameters - ---------- - force - If `True`, drop any objects that exist, and do not fail if the - database does not exist. - - """ - self.client.drop_database(self.name, force=force) - - def table(self, name: str) -> ir.Table: - """Return a table expression referencing a table in this database. - - Parameters - ---------- - name - The name of a table - - Returns - ------- - Table - Table expression - - """ - qualified_name = self._qualify(name) - return self.client.table(qualified_name, self.name) - - def list_tables(self, like=None, database=None): - """List the tables in the database. - - Parameters - ---------- - like - A pattern to use for listing tables. - database - The database to perform the list against - - """ - return self.client.list_tables(like, database=database or self.name) - - -class TablesAccessor(collections.abc.Mapping): - """A mapping-like object for accessing tables off a backend. - - Tables may be accessed by name using either index or attribute access: - - Examples - -------- - >>> con = ibis.sqlite.connect("example.db") - >>> people = con.tables["people"] # access via index - >>> people = con.tables.people # access via attribute - - """ - - def __init__(self, backend: BaseBackend): - self._backend = backend - - def __getitem__(self, name) -> ir.Table: - try: - return self._backend.table(name) - except Exception as exc: # noqa: BLE001 - raise KeyError(name) from exc - - def __getattr__(self, name) -> ir.Table: - if name.startswith("_"): - raise AttributeError(name) - try: - return self._backend.table(name) - except Exception as exc: # noqa: BLE001 - raise AttributeError(name) from exc - - def __iter__(self) -> Iterator[str]: - return iter(sorted(self._backend.list_tables())) - - def __len__(self) -> int: - return len(self._backend.list_tables()) - - def __dir__(self) -> list[str]: - o = set() - o.update(dir(type(self))) - o.update( - name - for name in self._backend.list_tables() - if name.isidentifier() and not keyword.iskeyword(name) - ) - return list(o) - - def __repr__(self) -> str: - tables = self._backend.list_tables() - rows = ["Tables", "------"] - rows.extend(f"- {name}" for name in sorted(tables)) - return "\n".join(rows) - - def _ipython_key_completions_(self) -> list[str]: - return self._backend.list_tables() - - -class _FileIOHandler: - @staticmethod - def _import_pyarrow(): - try: - import pyarrow # noqa: ICN001 - except ImportError: - raise ModuleNotFoundError( - "Exporting to arrow formats requires `pyarrow` but it is not installed" - ) - else: - import pyarrow_hotfix # noqa: F401 - - return pyarrow - - def to_pandas( - self, - expr: ir.Expr, - *, - params: Mapping[ir.Scalar, Any] | None = None, - limit: int | str | None = None, - **kwargs: Any, - ) -> pd.DataFrame | pd.Series | Any: - """Execute an Ibis expression and return a pandas `DataFrame`, `Series`, or scalar. - - ::: {.callout-note} - This method is a wrapper around `execute`. - ::: - - Parameters - ---------- - expr - Ibis expression to execute. - params - Mapping of scalar parameter expressions to value. - limit - An integer to effect a specific row limit. A value of `None` means - "no limit". The default is in `ibis/config.py`. - kwargs - Keyword arguments - - """ - return self.execute(expr, params=params, limit=limit, **kwargs) - - def to_pandas_batches( - self, - expr: ir.Expr, - *, - params: Mapping[ir.Scalar, Any] | None = None, - limit: int | str | None = None, - chunk_size: int = 1_000_000, - **kwargs: Any, - ) -> Iterator[pd.DataFrame | pd.Series | Any]: - """Execute an Ibis expression and return an iterator of pandas `DataFrame`s. - - Parameters - ---------- - expr - Ibis expression to execute. - params - Mapping of scalar parameter expressions to value. - limit - An integer to effect a specific row limit. A value of `None` means - "no limit". The default is in `ibis/config.py`. - chunk_size - Maximum number of rows in each returned `DataFrame` batch. This may have - no effect depending on the backend. - kwargs - Keyword arguments - - Returns - ------- - Iterator[pd.DataFrame] - An iterator of pandas `DataFrame`s. - - """ - from ibis.formats.pandas import PandasData - - orig_expr = expr - expr = expr.as_table() - schema = expr.schema() - yield from ( - orig_expr.__pandas_result__( - PandasData.convert_table(batch.to_pandas(), schema) - ) - for batch in self.to_pyarrow_batches( - expr, params=params, limit=limit, chunk_size=chunk_size, **kwargs - ) - ) - - @util.experimental - def to_pyarrow( - self, - expr: ir.Expr, - *, - params: Mapping[ir.Scalar, Any] | None = None, - limit: int | str | None = None, - **kwargs: Any, - ) -> pa.Table: - """Execute expression and return results in as a pyarrow table. - - This method is eager and will execute the associated expression - immediately. - - Parameters - ---------- - expr - Ibis expression to export to pyarrow - params - Mapping of scalar parameter expressions to value. - limit - An integer to effect a specific row limit. A value of `None` means - "no limit". The default is in `ibis/config.py`. - kwargs - Keyword arguments - - Returns - ------- - Table - A pyarrow table holding the results of the executed expression. - - """ - pa = self._import_pyarrow() - self._run_pre_execute_hooks(expr) - - table_expr = expr.as_table() - schema = table_expr.schema() - arrow_schema = schema.to_pyarrow() - with self.to_pyarrow_batches( - table_expr, params=params, limit=limit, **kwargs - ) as reader: - table = pa.Table.from_batches(reader, schema=arrow_schema) - - return expr.__pyarrow_result__( - table.rename_columns(table_expr.columns).cast(arrow_schema) - ) - - @util.experimental - def to_pyarrow_batches( - self, - expr: ir.Expr, - *, - params: Mapping[ir.Scalar, Any] | None = None, - limit: int | str | None = None, - chunk_size: int = 1_000_000, - **kwargs: Any, - ) -> pa.ipc.RecordBatchReader: - """Execute expression and return a RecordBatchReader. - - This method is eager and will execute the associated expression - immediately. - - Parameters - ---------- - expr - Ibis expression to export to pyarrow - limit - An integer to effect a specific row limit. A value of `None` means - "no limit". The default is in `ibis/config.py`. - params - Mapping of scalar parameter expressions to value. - chunk_size - Maximum number of rows in each returned record batch. - kwargs - Keyword arguments - - Returns - ------- - results - RecordBatchReader - - """ - raise NotImplementedError - - @util.experimental - def to_torch( - self, - expr: ir.Expr, - *, - params: Mapping[ir.Scalar, Any] | None = None, - limit: int | str | None = None, - **kwargs: Any, - ) -> dict[str, torch.Tensor]: - """Execute an expression and return results as a dictionary of torch tensors. - - Parameters - ---------- - expr - Ibis expression to execute. - params - Parameters to substitute into the expression. - limit - An integer to effect a specific row limit. A value of `None` means no limit. - kwargs - Keyword arguments passed into the backend's `to_torch` implementation. - - Returns - ------- - dict[str, torch.Tensor] - A dictionary of torch tensors, keyed by column name. - - """ - import torch - - t = self.to_pyarrow(expr, params=params, limit=limit, **kwargs) - # without .copy() the arrays are read-only and thus writing to them is - # undefined behavior; we can't ignore this warning from torch because - # we're going out of ibis and downstream code can do whatever it wants - # with the data - return { - name: torch.from_numpy(t[name].to_numpy().copy()) for name in t.schema.names - } - - def read_parquet( - self, path: str | Path, table_name: str | None = None, **kwargs: Any - ) -> ir.Table: - """Register a parquet file as a table in the current backend. - - Parameters - ---------- - path - The data source. - table_name - An optional name to use for the created table. This defaults to - a sequentially generated name. - **kwargs - Additional keyword arguments passed to the backend loading function. - - Returns - ------- - ir.Table - The just-registered table - - """ - raise NotImplementedError( - f"{self.name} does not support direct registration of parquet data." - ) - - def read_csv( - self, path: str | Path, table_name: str | None = None, **kwargs: Any - ) -> ir.Table: - """Register a CSV file as a table in the current backend. - - Parameters - ---------- - path - The data source. A string or Path to the CSV file. - table_name - An optional name to use for the created table. This defaults to - a sequentially generated name. - **kwargs - Additional keyword arguments passed to the backend loading function. - - Returns - ------- - ir.Table - The just-registered table - - """ - raise NotImplementedError( - f"{self.name} does not support direct registration of CSV data." - ) - - def read_json( - self, path: str | Path, table_name: str | None = None, **kwargs: Any - ) -> ir.Table: - """Register a JSON file as a table in the current backend. - - Parameters - ---------- - path - The data source. A string or Path to the JSON file. - table_name - An optional name to use for the created table. This defaults to - a sequentially generated name. - **kwargs - Additional keyword arguments passed to the backend loading function. - - Returns - ------- - ir.Table - The just-registered table - - """ - raise NotImplementedError( - f"{self.name} does not support direct registration of JSON data." - ) - - def read_delta( - self, source: str | Path, table_name: str | None = None, **kwargs: Any - ): - """Register a Delta Lake table in the current database. - - Parameters - ---------- - source - The data source. Must be a directory - containing a Delta Lake table. - table_name - An optional name to use for the created table. This defaults to - a sequentially generated name. - **kwargs - Additional keyword arguments passed to the underlying backend or library. - - Returns - ------- - ir.Table - The just-registered table. - - """ - raise NotImplementedError( - f"{self.name} does not support direct registration of DeltaLake tables." - ) - - @util.experimental - def to_parquet( - self, - expr: ir.Table, - path: str | Path, - *, - params: Mapping[ir.Scalar, Any] | None = None, - **kwargs: Any, - ) -> None: - """Write the results of executing the given expression to a parquet file. - - This method is eager and will execute the associated expression - immediately. - - Parameters - ---------- - expr - The ibis expression to execute and persist to parquet. - path - The data source. A string or Path to the parquet file. - params - Mapping of scalar parameter expressions to value. - **kwargs - Additional keyword arguments passed to pyarrow.parquet.ParquetWriter - - https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html - - """ - self._import_pyarrow() - import pyarrow.parquet as pq - - with expr.to_pyarrow_batches(params=params) as batch_reader: - with pq.ParquetWriter(path, batch_reader.schema, **kwargs) as writer: - for batch in batch_reader: - writer.write_batch(batch) - - @util.experimental - def to_csv( - self, - expr: ir.Table, - path: str | Path, - *, - params: Mapping[ir.Scalar, Any] | None = None, - **kwargs: Any, - ) -> None: - """Write the results of executing the given expression to a CSV file. - - This method is eager and will execute the associated expression - immediately. - - Parameters - ---------- - expr - The ibis expression to execute and persist to CSV. - path - The data source. A string or Path to the CSV file. - params - Mapping of scalar parameter expressions to value. - kwargs - Additional keyword arguments passed to pyarrow.csv.CSVWriter - - https://arrow.apache.org/docs/python/generated/pyarrow.csv.CSVWriter.html - - """ - self._import_pyarrow() - import pyarrow.csv as pcsv - - with expr.to_pyarrow_batches(params=params) as batch_reader: - with pcsv.CSVWriter(path, batch_reader.schema, **kwargs) as writer: - for batch in batch_reader: - writer.write_batch(batch) - - @util.experimental - def to_delta( - self, - expr: ir.Table, - path: str | Path, - *, - params: Mapping[ir.Scalar, Any] | None = None, - **kwargs: Any, - ) -> None: - """Write the results of executing the given expression to a Delta Lake table. - - This method is eager and will execute the associated expression - immediately. - - Parameters - ---------- - expr - The ibis expression to execute and persist to Delta Lake table. - path - The data source. A string or Path to the Delta Lake table. - params - Mapping of scalar parameter expressions to value. - kwargs - Additional keyword arguments passed to deltalake.writer.write_deltalake method - - """ - try: - from deltalake.writer import write_deltalake - except ImportError: - raise ImportError( - "The deltalake extra is required to use the " - "to_delta method. You can install it using pip:\n\n" - "pip install 'ibis-framework[deltalake]'\n" - ) - - with expr.to_pyarrow_batches(params=params) as batch_reader: - write_deltalake(path, batch_reader, **kwargs) - - -class CanListDatabases(abc.ABC): - @abc.abstractmethod - def list_databases(self, like: str | None = None) -> list[str]: - """List existing databases in the current connection. - - Parameters - ---------- - like - A pattern in Python's regex format to filter returned database - names. - - Returns - ------- - list[str] - The database names that exist in the current connection, that match - the `like` pattern if provided. - - """ - - @property - @abc.abstractmethod - def current_database(self) -> str: - """The current database in use.""" - - -class CanCreateDatabase(CanListDatabases): - @abc.abstractmethod - def create_database(self, name: str, force: bool = False) -> None: - """Create a new database. - - Parameters - ---------- - name - Name of the new database. - force - If `False`, an exception is raised if the database already exists. - - """ - - @abc.abstractmethod - def drop_database(self, name: str, force: bool = False) -> None: - """Drop a database with name `name`. - - Parameters - ---------- - name - Database to drop. - force - If `False`, an exception is raised if the database does not exist. - - """ - - -class CanCreateSchema(abc.ABC): - @abc.abstractmethod - def create_schema( - self, name: str, database: str | None = None, force: bool = False - ) -> None: - """Create a schema named `name` in `database`. - - Parameters - ---------- - name - Name of the schema to create. - database - Name of the database in which to create the schema. If `None`, the - current database is used. - force - If `False`, an exception is raised if the schema exists. - - """ - - @abc.abstractmethod - def drop_schema( - self, name: str, database: str | None = None, force: bool = False - ) -> None: - """Drop the schema with `name` in `database`. - - Parameters - ---------- - name - Name of the schema to drop. - database - Name of the database to drop the schema from. If `None`, the - current database is used. - force - If `False`, an exception is raised if the schema does not exist. - - """ - - @abc.abstractmethod - def list_schemas( - self, like: str | None = None, database: str | None = None - ) -> list[str]: - """List existing schemas in the current connection. - - Parameters - ---------- - like - A pattern in Python's regex format to filter returned schema - names. - database - The database to list schemas from. If `None`, the current database - is searched. - - Returns - ------- - list[str] - The schema names that exist in the current connection, that match - the `like` pattern if provided. - - """ - - @property - @abc.abstractmethod - def current_schema(self) -> str: - """Return the current schema.""" - - -class BaseBackend(abc.ABC, _FileIOHandler): - """Base backend class. - - All Ibis backends must subclass this class and implement all the - required methods. - """ - - name: ClassVar[str] - - supports_temporary_tables = False - supports_python_udfs = False - supports_in_memory_tables = True - - def __init__(self, *args, **kwargs): - self._con_args: tuple[Any] = args - self._con_kwargs: dict[str, Any] = kwargs - # expression cache - self._query_cache = RefCountedCache( - populate=self._load_into_cache, - lookup=lambda name: self.table(name).op(), - finalize=self._clean_up_cached_table, - generate_name=functools.partial(util.gen_name, "cache"), - key=lambda expr: expr.op(), - ) - - @property - @abc.abstractmethod - def dialect(self) -> sg.Dialect | None: - """The sqlglot dialect for this backend, where applicable. - - Returns None if the backend is not a SQL backend. - """ - - def __getstate__(self): - return dict(_con_args=self._con_args, _con_kwargs=self._con_kwargs) - - def __rich_repr__(self): - yield "name", self.name - - def __hash__(self): - return hash(self.db_identity) - - def __eq__(self, other): - return self.db_identity == other.db_identity - - @functools.cached_property - def db_identity(self) -> str: - """Return the identity of the database. - - Multiple connections to the same - database will return the same value for `db_identity`. - - The default implementation assumes connection parameters uniquely - specify the database. - - Returns - ------- - Hashable - Database identity - - """ - parts = [self.__class__] - parts.extend(self._con_args) - parts.extend(f"{k}={v}" for k, v in self._con_kwargs.items()) - return "_".join(map(str, parts)) - - # TODO(kszucs): this should be a classmethod returning with a new backend - # instance which does instantiate the connection - def connect(self, *args, **kwargs) -> BaseBackend: - """Connect to the database. - - Parameters - ---------- - *args - Mandatory connection parameters, see the docstring of `do_connect` - for details. - **kwargs - Extra connection parameters, see the docstring of `do_connect` for - details. - - Notes - ----- - This creates a new backend instance with saved `args` and `kwargs`, - then calls `reconnect` and finally returns the newly created and - connected backend instance. - - Returns - ------- - BaseBackend - An instance of the backend - - """ - new_backend = self.__class__(*args, **kwargs) - new_backend.reconnect() - return new_backend - - @abc.abstractmethod - def disconnect(self) -> None: - """Close the connection to the backend.""" - - @staticmethod - def _convert_kwargs(kwargs: MutableMapping) -> None: - """Manipulate keyword arguments to `.connect` method.""" - - # TODO(kszucs): should call self.connect(*self._con_args, **self._con_kwargs) - def reconnect(self) -> None: - """Reconnect to the database already configured with connect.""" - self.do_connect(*self._con_args, **self._con_kwargs) - - def do_connect(self, *args, **kwargs) -> None: - """Connect to database specified by `args` and `kwargs`.""" - - @util.deprecated(instead="use equivalent methods in the backend") - def database(self, name: str | None = None) -> Database: - """Return a `Database` object for the `name` database. - - Parameters - ---------- - name - Name of the database to return the object for. - - Returns - ------- - Database - A database object for the specified database. - - """ - return Database(name=name or self.current_database, client=self) - - @staticmethod - def _filter_with_like(values: Iterable[str], like: str | None = None) -> list[str]: - """Filter names with a `like` pattern (regex). - - The methods `list_databases` and `list_tables` accept a `like` - argument, which filters the returned tables with tables that match the - provided pattern. - - We provide this method in the base backend, so backends can use it - instead of reinventing the wheel. - - Parameters - ---------- - values - Iterable of strings to filter - like - Pattern to use for filtering names - - Returns - ------- - list[str] - Names filtered by the `like` pattern. - - """ - if like is None: - return sorted(values) - - pattern = re.compile(like) - return sorted(filter(pattern.findall, values)) - - @abc.abstractmethod - def list_tables( - self, like: str | None = None, database: str | None = None - ) -> list[str]: - """Return the list of table names in the current database. - - For some backends, the tables may be files in a directory, - or other equivalent entities in a SQL database. - - Parameters - ---------- - like - A pattern in Python's regex format. - database - The database from which to list tables. If not provided, the - current database is used. - - Returns - ------- - list[str] - The list of the table names that match the pattern `like`. - - """ - - @abc.abstractmethod - def table(self, name: str, database: str | None = None) -> ir.Table: - """Construct a table expression. - - Parameters - ---------- - name - Table name - database - Database name - - Returns - ------- - Table - Table expression - - """ - - @functools.cached_property - def tables(self): - """An accessor for tables in the database. - - Tables may be accessed by name using either index or attribute access: - - Examples - -------- - >>> con = ibis.sqlite.connect("example.db") - >>> people = con.tables["people"] # access via index - >>> people = con.tables.people # access via attribute - - """ - return TablesAccessor(self) - - @property - @abc.abstractmethod - def version(self) -> str: - """Return the version of the backend engine. - - For database servers, return the server version. - - For others such as SQLite and pandas return the version of the - underlying library or application. - - Returns - ------- - str - The backend version - - """ - - @classmethod - def register_options(cls) -> None: - """Register custom backend options.""" - options = ibis.config.options - backend_name = cls.name - try: - backend_options = cls.Options() - except AttributeError: - pass - else: - try: - setattr(options, backend_name, backend_options) - except ValueError as e: - raise exc.BackendConfigurationNotRegistered(backend_name) from e - - def _register_udfs(self, expr: ir.Expr) -> None: - """Register UDFs contained in `expr` with the backend.""" - if self.supports_python_udfs: - raise NotImplementedError(self.name) - - def _register_in_memory_tables(self, expr: ir.Expr): - if self.supports_in_memory_tables: - raise NotImplementedError(self.name) - - def _run_pre_execute_hooks(self, expr: ir.Expr) -> None: - """Backend-specific hooks to run before an expression is executed.""" - self._define_udf_translation_rules(expr) - self._register_udfs(expr) - self._register_in_memory_tables(expr) - - def _define_udf_translation_rules(self, expr: ir.Expr): - if self.supports_python_udfs: - raise NotImplementedError(self.name) - - def compile( - self, - expr: ir.Expr, - params: Mapping[ir.Expr, Any] | None = None, - ) -> Any: - """Compile an expression.""" - return self.compiler.to_sql(expr, params=params) - - def _to_sql(self, expr: ir.Expr, **kwargs) -> str: - """Convert an expression to a SQL string. - - Called by `ibis.to_sql`; gives the backend an opportunity to generate - nicer SQL for human consumption. - """ - raise NotImplementedError(f"Backend '{self.name}' backend doesn't support SQL") - - def execute(self, expr: ir.Expr) -> Any: - """Execute an expression.""" - - def add_operation(self, operation: ops.Node) -> Callable: - """Add a translation function to the backend for a specific operation. - - Operations are defined in `ibis.expr.operations`, and a translation - function receives the translator object and an expression as - parameters, and returns a value depending on the backend. - """ - if not hasattr(self, "compiler"): - raise RuntimeError("Only SQL-based backends support `add_operation`") - - def decorator(translation_function: Callable) -> None: - self.compiler.translator_class.add_operation( - operation, translation_function - ) - - return decorator - - @abc.abstractmethod - def create_table( - self, - name: str, - obj: pd.DataFrame | pa.Table | ir.Table | None = None, - *, - schema: ibis.Schema | None = None, - database: str | None = None, - temp: bool = False, - overwrite: bool = False, - ) -> ir.Table: - """Create a new table. - - Parameters - ---------- - name - Name of the new table. - obj - An Ibis table expression or pandas table that will be used to - extract the schema and the data of the new table. If not provided, - `schema` must be given. - schema - The schema for the new table. Only one of `schema` or `obj` can be - provided. - database - Name of the database where the table will be created, if not the - default. - temp - Whether a table is temporary or not - overwrite - Whether to clobber existing data - - Returns - ------- - Table - The table that was created. - - """ - - @abc.abstractmethod - def drop_table( - self, - name: str, - *, - database: str | None = None, - force: bool = False, - ) -> None: - """Drop a table. - - Parameters - ---------- - name - Name of the table to drop. - database - Name of the database where the table exists, if not the default. - force - If `False`, an exception is raised if the table does not exist. - - """ - raise NotImplementedError( - f'Backend "{self.name}" does not implement "drop_table"' - ) - - def rename_table(self, old_name: str, new_name: str) -> None: - """Rename an existing table. - - Parameters - ---------- - old_name - The old name of the table. - new_name - The new name of the table. - - """ - raise NotImplementedError( - f'Backend "{self.name}" does not implement "rename_table"' - ) - - @abc.abstractmethod - def create_view( - self, - name: str, - obj: ir.Table, - *, - database: str | None = None, - overwrite: bool = False, - ) -> ir.Table: - """Create a new view from an expression. - - Parameters - ---------- - name - Name of the new view. - obj - An Ibis table expression that will be used to create the view. - database - Name of the database where the view will be created, if not - provided the database's default is used. - overwrite - Whether to clobber an existing view with the same name - - Returns - ------- - Table - The view that was created. - - """ - - @abc.abstractmethod - def drop_view( - self, name: str, *, database: str | None = None, force: bool = False - ) -> None: - """Drop a view. - - Parameters - ---------- - name - Name of the view to drop. - database - Name of the database where the view exists, if not the default. - force - If `False`, an exception is raised if the view does not exist. - - """ - - @classmethod - def has_operation(cls, operation: type[ops.Value]) -> bool: - """Return whether the backend implements support for `operation`. - - Parameters - ---------- - operation - A class corresponding to an operation. - - Returns - ------- - bool - Whether the backend implements the operation. - - Examples - -------- - >>> import ibis - >>> import ibis.expr.operations as ops - >>> ibis.sqlite.has_operation(ops.ArrayIndex) - False - >>> ibis.postgres.has_operation(ops.ArrayIndex) - True - - """ - raise NotImplementedError( - f"{cls.name} backend has not implemented `has_operation` API" - ) - - def _cached(self, expr: ir.Table): - """Cache the provided expression. - - All subsequent operations on the returned expression will be performed on the cached data. - - Parameters - ---------- - expr - Table expression to cache - - Returns - ------- - Expr - Cached table - - """ - op = expr.op() - if (result := self._query_cache.get(op)) is None: - self._query_cache.store(expr) - result = self._query_cache[op] - return ir.CachedTable(result) - - def _release_cached(self, expr: ir.CachedTable) -> None: - """Releases the provided cached expression. - - Parameters - ---------- - expr - Cached expression to release - - """ - del self._query_cache[expr.op()] - - def _load_into_cache(self, name, expr): - raise NotImplementedError(self.name) - - def _clean_up_cached_table(self, op): - raise NotImplementedError(self.name) - - def _transpile_sql(self, query: str, *, dialect: str | None = None) -> str: - # only transpile if dialect was passed - if dialect is None: - return query - - import sqlglot as sg - - # only transpile if the backend dialect doesn't match the input dialect - name = self.name - if (output_dialect := self.dialect) is None: - raise NotImplementedError(f"No known sqlglot dialect for backend {name}") - - if dialect != output_dialect: - (query,) = sg.transpile(query, read=dialect, write=output_dialect) - return query - - -@functools.cache -def _get_backend_names(*, exclude: tuple[str] = ()) -> frozenset[str]: - """Return the set of known backend names. - - Parameters - ---------- - exclude - Exclude these backend names from the result - - Notes - ----- - This function returns a frozenset to prevent cache pollution. - - If a `set` is used, then any in-place modifications to the set - are visible to every caller of this function. - - """ - - if sys.version_info < (3, 10): - entrypoints = importlib.metadata.entry_points()["ibis.backends"] - else: - entrypoints = importlib.metadata.entry_points(group="ibis.backends") - return frozenset(ep.name for ep in entrypoints).difference(exclude) - - -def connect(resource: Path | str, **kwargs: Any) -> BaseBackend: - """Connect to `resource`, inferring the backend automatically. - - The general pattern for `ibis.connect` is - - ```python - con = ibis.connect("backend://connection-parameters") - ``` - - With many backends that looks like - - ```python - con = ibis.connect("backend://user:password@host:port/database") - ``` - - See the connection syntax for each backend for details about URL connection - requirements. - - Parameters - ---------- - resource - A URL or path to the resource to be connected to. - kwargs - Backend specific keyword arguments - - Examples - -------- - Connect to an in-memory DuckDB database: - - >>> import ibis - >>> con = ibis.connect("duckdb://") - - Connect to an on-disk SQLite database: - - >>> con = ibis.connect("sqlite://relative.db") - >>> con = ibis.connect( - ... "sqlite:///absolute/path/to/data.db" - ... ) # quartodoc: +SKIP # doctest: +SKIP - - Connect to a PostgreSQL server: - - >>> con = ibis.connect( - ... "postgres://user:password@hostname:5432" - ... ) # quartodoc: +SKIP # doctest: +SKIP - - Connect to BigQuery: - - >>> con = ibis.connect( - ... "bigquery://my-project/my-dataset" - ... ) # quartodoc: +SKIP # doctest: +SKIP - - """ - url = resource = str(resource) - - if re.match("[A-Za-z]:", url): - # windows path with drive, treat it as a file - url = f"file://{url}" - - parsed = urllib.parse.urlparse(url) - scheme = parsed.scheme or "file" - - orig_kwargs = kwargs.copy() - kwargs = dict(urllib.parse.parse_qsl(parsed.query)) - - if scheme == "file": - path = parsed.netloc + parsed.path - # Merge explicit kwargs with query string, explicit kwargs - # taking precedence - kwargs.update(orig_kwargs) - if path.endswith(".duckdb"): - return ibis.duckdb.connect(path, **kwargs) - elif path.endswith((".sqlite", ".db")): - return ibis.sqlite.connect(path, **kwargs) - elif path.endswith((".parquet", ".csv", ".csv.gz")): - # Load parquet/csv/csv.gz files with duckdb by default - con = ibis.duckdb.connect(**kwargs) - con.register(path) - return con - else: - raise ValueError(f"Don't know how to connect to {resource!r}") - - if kwargs: - # If there are kwargs (either explicit or from the query string), - # re-add them to the parsed URL - query = urllib.parse.urlencode(kwargs) - parsed = parsed._replace(query=query) - - if scheme in ("postgres", "postgresql"): - # Treat `postgres://` and `postgresql://` the same - scheme = "postgres" - - # Convert all arguments back to a single URL string - url = parsed.geturl() - if "://" not in url: - # urllib may roundtrip `duckdb://` to `duckdb:`. Here we re-add the - # missing `//`. - url = url.replace(":", "://", 1) - - try: - backend = getattr(ibis, scheme) - except AttributeError: - raise ValueError(f"Don't know how to connect to {resource!r}") from None - - return backend._from_url(url, **orig_kwargs) - - -class UrlFromPath: - __slots__ = () - - def _from_url(self, url: str, **kwargs) -> BaseBackend: - """Connect to a backend using a URL `url`. - - Parameters - ---------- - url - URL with which to connect to a backend. - kwargs - Additional keyword arguments - - Returns - ------- - BaseBackend - A backend instance - - """ - url = urlparse(url) - netloc = url.netloc - parts = list(filter(None, (netloc, url.path[bool(netloc) :]))) - database = Path(*parts) if parts and parts != [":memory:"] else ":memory:" - if (strdatabase := str(database)).startswith("md:") or strdatabase.startswith( - "motherduck:" - ): - database = strdatabase - elif isinstance(database, Path): - database = database.absolute() - - query_params = parse_qs(url.query) - - for name, value in query_params.items(): - if len(value) > 1: - kwargs[name] = value - elif len(value) == 1: - kwargs[name] = value[0] - else: - raise exc.IbisError(f"Invalid URL parameter: {name}") - - self._convert_kwargs(kwargs) - return self.connect(database=database, **kwargs) - - -class NoUrl: - __slots__ = () - - name: str - - def _from_url(self, url: str, **_) -> ir.Table: - raise NotImplementedError(self.name) diff --git a/ibis/backends/bigquery/__init__.py b/ibis/backends/bigquery/__init__.py index 7ddbd7671e8e..64c9a780d266 100644 --- a/ibis/backends/bigquery/__init__.py +++ b/ibis/backends/bigquery/__init__.py @@ -24,9 +24,7 @@ import ibis.expr.operations as ops import ibis.expr.types as ir from ibis import util -from ibis.backends.base import CanCreateSchema, Database -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.datatypes import BigQueryType +from ibis.backends import CanCreateSchema, Database from ibis.backends.bigquery.client import ( BigQueryCursor, bigquery_param, @@ -37,6 +35,8 @@ from ibis.backends.bigquery.compiler import BigQueryCompiler from ibis.backends.bigquery.datatypes import BigQuerySchema from ibis.backends.bigquery.udf.core import PythonToJavaScriptTranslator +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.datatypes import BigQueryType if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping @@ -123,7 +123,7 @@ def _remove_null_ordering_from_unsupported_window( return node -class Backend(SQLGlotBackend, CanCreateSchema): +class Backend(SQLBackend, CanCreateSchema): name = "bigquery" compiler = BigQueryCompiler() supports_in_memory_tables = True diff --git a/ibis/backends/bigquery/compiler.py b/ibis/backends/bigquery/compiler.py index e4cfdfff298c..da2c0de87bfb 100644 --- a/ibis/backends/bigquery/compiler.py +++ b/ibis/backends/bigquery/compiler.py @@ -12,9 +12,9 @@ import ibis.expr.datatypes as dt import ibis.expr.operations as ops from ibis import util -from ibis.backends.base.sqlglot.compiler import NULL, STAR, SQLGlotCompiler, paren -from ibis.backends.base.sqlglot.datatypes import BigQueryType, BigQueryUDFType -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler, paren +from ibis.backends.sql.datatypes import BigQueryType, BigQueryUDFType +from ibis.backends.sql.rewrites import ( exclude_unsupported_window_frame_from_ops, exclude_unsupported_window_frame_from_rank, exclude_unsupported_window_frame_from_row_number, diff --git a/ibis/backends/bigquery/datatypes.py b/ibis/backends/bigquery/datatypes.py index 5ddb21c46e39..5af92df355f7 100644 --- a/ibis/backends/bigquery/datatypes.py +++ b/ibis/backends/bigquery/datatypes.py @@ -5,7 +5,7 @@ import ibis import ibis.expr.datatypes as dt import ibis.expr.schema as sch -from ibis.backends.base.sqlglot.datatypes import BigQueryType +from ibis.backends.sql.datatypes import BigQueryType from ibis.formats import SchemaMapper diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index 43f490a07f4d..781d5d0e4534 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -23,10 +23,10 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import BaseBackend, CanCreateDatabase -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import C +from ibis.backends import BaseBackend, CanCreateDatabase from ibis.backends.clickhouse.compiler import ClickHouseCompiler +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import C if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping @@ -39,7 +39,7 @@ def _to_memtable(v): return ibis.memtable(v).op() if not isinstance(v, ops.InMemoryTable) else v -class Backend(SQLGlotBackend, CanCreateDatabase): +class Backend(SQLBackend, CanCreateDatabase): name = "clickhouse" compiler = ClickHouseCompiler() diff --git a/ibis/backends/clickhouse/compiler.py b/ibis/backends/clickhouse/compiler.py index d94fe936763f..3ca46e925e58 100644 --- a/ibis/backends/clickhouse/compiler.py +++ b/ibis/backends/clickhouse/compiler.py @@ -11,15 +11,15 @@ import ibis.expr.datatypes as dt import ibis.expr.operations as ops from ibis import util -from ibis.backends.base.sqlglot.compiler import ( +from ibis.backends.sql.compiler import ( NULL, STAR, SQLGlotCompiler, parenthesize, ) -from ibis.backends.base.sqlglot.datatypes import ClickHouseType -from ibis.backends.base.sqlglot.dialects import ClickHouse -from ibis.backends.base.sqlglot.rewrites import rewrite_sample_as_filter +from ibis.backends.sql.datatypes import ClickHouseType +from ibis.backends.sql.dialects import ClickHouse +from ibis.backends.sql.rewrites import rewrite_sample_as_filter class ClickHouseCompiler(SQLGlotCompiler): diff --git a/ibis/backends/clickhouse/tests/test_datatypes.py b/ibis/backends/clickhouse/tests/test_datatypes.py index 69c10c432798..ae85290cde98 100644 --- a/ibis/backends/clickhouse/tests/test_datatypes.py +++ b/ibis/backends/clickhouse/tests/test_datatypes.py @@ -8,7 +8,7 @@ import ibis import ibis.expr.datatypes as dt import ibis.tests.strategies as its -from ibis.backends.base.sqlglot.datatypes import ClickHouseType +from ibis.backends.sql.datatypes import ClickHouseType pytest.importorskip("clickhouse_connect") diff --git a/ibis/backends/conftest.py b/ibis/backends/conftest.py index e48fdc5cb5ac..c089475ee747 100644 --- a/ibis/backends/conftest.py +++ b/ibis/backends/conftest.py @@ -19,7 +19,7 @@ import ibis import ibis.common.exceptions as com from ibis import util -from ibis.backends.base import CanCreateDatabase, CanCreateSchema, _get_backend_names +from ibis.backends import CanCreateDatabase, CanCreateSchema, _get_backend_names from ibis.conftest import WINDOWS from ibis.util import promote_tuple @@ -602,7 +602,7 @@ def temp_table(con) -> str: Parameters ---------- - con : ibis.backends.base.Client + con : ibis.backends.Client Yields ------ @@ -657,7 +657,7 @@ def alternate_current_database(ddl_con, ddl_backend) -> str: Parameters ---------- - ddl_con : ibis.backends.base.Client + ddl_con : ibis.backends.Client Yields ------ diff --git a/ibis/backends/dask/__init__.py b/ibis/backends/dask/__init__.py index 67ee86b89dea..7937bc55f42f 100644 --- a/ibis/backends/dask/__init__.py +++ b/ibis/backends/dask/__init__.py @@ -14,7 +14,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import NoUrl +from ibis.backends import NoUrl from ibis.backends.pandas import BasePandasBackend from ibis.formats.pandas import PandasData diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index ee251cee9f39..f4d4dd22ceda 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -20,10 +20,10 @@ import ibis.expr.operations as ops import ibis.expr.schema as sch import ibis.expr.types as ir -from ibis.backends.base import CanCreateDatabase, CanCreateSchema, NoUrl -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import C +from ibis.backends import CanCreateDatabase, CanCreateSchema, NoUrl from ibis.backends.datafusion.compiler import DataFusionCompiler +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import C from ibis.expr.operations.udf import InputType from ibis.formats.pyarrow import PyArrowType from ibis.util import gen_name, normalize_filename @@ -44,7 +44,7 @@ import pandas as pd -class Backend(SQLGlotBackend, CanCreateDatabase, CanCreateSchema, NoUrl): +class Backend(SQLBackend, CanCreateDatabase, CanCreateSchema, NoUrl): name = "datafusion" supports_in_memory_tables = True supports_arrays = True diff --git a/ibis/backends/datafusion/compiler.py b/ibis/backends/datafusion/compiler.py index 980936288878..f618e332da7c 100644 --- a/ibis/backends/datafusion/compiler.py +++ b/ibis/backends/datafusion/compiler.py @@ -11,16 +11,16 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import ( +from ibis.backends.sql.compiler import ( FALSE, NULL, STAR, SQLGlotCompiler, paren, ) -from ibis.backends.base.sqlglot.datatypes import DataFusionType -from ibis.backends.base.sqlglot.dialects import DataFusion -from ibis.backends.base.sqlglot.rewrites import rewrite_sample_as_filter +from ibis.backends.sql.datatypes import DataFusionType +from ibis.backends.sql.dialects import DataFusion +from ibis.backends.sql.rewrites import rewrite_sample_as_filter from ibis.common.temporal import IntervalUnit, TimestampUnit from ibis.expr.operations.udf import InputType from ibis.formats.pyarrow import PyArrowType diff --git a/ibis/backends/druid/__init__.py b/ibis/backends/druid/__init__.py index 411c83a4866d..c002a014677e 100644 --- a/ibis/backends/druid/__init__.py +++ b/ibis/backends/druid/__init__.py @@ -13,10 +13,10 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.schema as sch -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import STAR -from ibis.backends.base.sqlglot.datatypes import DruidType from ibis.backends.druid.compiler import DruidCompiler +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import STAR +from ibis.backends.sql.datatypes import DruidType if TYPE_CHECKING: from collections.abc import Iterable, Mapping @@ -27,7 +27,7 @@ import ibis.expr.types as ir -class Backend(SQLGlotBackend): +class Backend(SQLBackend): name = "druid" compiler = DruidCompiler() supports_create_or_replace = False diff --git a/ibis/backends/druid/compiler.py b/ibis/backends/druid/compiler.py index 30cbbcae1ec4..da3453ffdb26 100644 --- a/ibis/backends/druid/compiler.py +++ b/ibis/backends/druid/compiler.py @@ -6,10 +6,10 @@ import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import NULL, SQLGlotCompiler -from ibis.backends.base.sqlglot.datatypes import DruidType -from ibis.backends.base.sqlglot.dialects import Druid -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, SQLGlotCompiler +from ibis.backends.sql.datatypes import DruidType +from ibis.backends.sql.dialects import Druid +from ibis.backends.sql.rewrites import ( rewrite_capitalize, rewrite_sample_as_filter, ) diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index aeb4a3a10a45..6088c9734dc6 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -23,11 +23,11 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import CanCreateSchema, UrlFromPath -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import STAR, C, F +from ibis.backends import CanCreateSchema, UrlFromPath from ibis.backends.duckdb.compiler import DuckDBCompiler from ibis.backends.duckdb.converter import DuckDBPandasData +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import STAR, C, F from ibis.expr.operations.udf import InputType if TYPE_CHECKING: @@ -74,7 +74,7 @@ def __repr__(self): return repr(dict(zip(kv["key"], kv["value"]))) -class Backend(SQLGlotBackend, CanCreateSchema, UrlFromPath): +class Backend(SQLBackend, CanCreateSchema, UrlFromPath): name = "duckdb" compiler = DuckDBCompiler() diff --git a/ibis/backends/duckdb/compiler.py b/ibis/backends/duckdb/compiler.py index 6bcb0273f313..208993fe8846 100644 --- a/ibis/backends/duckdb/compiler.py +++ b/ibis/backends/duckdb/compiler.py @@ -11,12 +11,12 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import ( +from ibis.backends.sql.compiler import ( NULL, STAR, SQLGlotCompiler, ) -from ibis.backends.base.sqlglot.datatypes import DuckDBType +from ibis.backends.sql.datatypes import DuckDBType _INTERVAL_SUFFIXES = { "ms": "milliseconds", diff --git a/ibis/backends/duckdb/tests/conftest.py b/ibis/backends/duckdb/tests/conftest.py index 62de7ed3e61e..a31f138274c6 100644 --- a/ibis/backends/duckdb/tests/conftest.py +++ b/ibis/backends/duckdb/tests/conftest.py @@ -12,7 +12,7 @@ if TYPE_CHECKING: from collections.abc import Iterator - from ibis.backends.base import BaseBackend + from ibis.backends import BaseBackend TEST_TABLES_GEO = { "zones": ibis.schema( diff --git a/ibis/backends/duckdb/tests/test_datatypes.py b/ibis/backends/duckdb/tests/test_datatypes.py index 2effe21bd644..8039c139936c 100644 --- a/ibis/backends/duckdb/tests/test_datatypes.py +++ b/ibis/backends/duckdb/tests/test_datatypes.py @@ -6,7 +6,7 @@ import ibis import ibis.expr.datatypes as dt -from ibis.backends.base.sqlglot.datatypes import DuckDBType +from ibis.backends.sql.datatypes import DuckDBType @pytest.mark.parametrize( diff --git a/ibis/backends/exasol/__init__.py b/ibis/backends/exasol/__init__.py index f72b1d046dc0..6c4e5e751721 100644 --- a/ibis/backends/exasol/__init__.py +++ b/ibis/backends/exasol/__init__.py @@ -18,9 +18,9 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import STAR, C from ibis.backends.exasol.compiler import ExasolCompiler +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import STAR, C if TYPE_CHECKING: from collections.abc import Iterable, Mapping @@ -28,13 +28,13 @@ import pandas as pd import pyarrow as pa - from ibis.backends.base import BaseBackend + from ibis.backends import BaseBackend # strip trailing encodings e.g., UTF8 _VARCHAR_REGEX = re.compile(r"^((VAR)?CHAR(?:\(\d+\)))?(?:\s+.+)?$") -class Backend(SQLGlotBackend): +class Backend(SQLBackend): name = "exasol" compiler = ExasolCompiler() supports_temporary_tables = False diff --git a/ibis/backends/exasol/compiler.py b/ibis/backends/exasol/compiler.py index 821ddcfea037..369a9cc444db 100644 --- a/ibis/backends/exasol/compiler.py +++ b/ibis/backends/exasol/compiler.py @@ -6,10 +6,10 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import NULL, SQLGlotCompiler -from ibis.backends.base.sqlglot.datatypes import ExasolType -from ibis.backends.base.sqlglot.dialects import Exasol -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, SQLGlotCompiler +from ibis.backends.sql.datatypes import ExasolType +from ibis.backends.sql.dialects import Exasol +from ibis.backends.sql.rewrites import ( exclude_unsupported_window_frame_from_ops, exclude_unsupported_window_frame_from_rank, exclude_unsupported_window_frame_from_row_number, diff --git a/ibis/backends/flink/__init__.py b/ibis/backends/flink/__init__.py index 373fb8f377b7..21ce76a81d36 100644 --- a/ibis/backends/flink/__init__.py +++ b/ibis/backends/flink/__init__.py @@ -10,8 +10,7 @@ import ibis.expr.operations as ops import ibis.expr.schema as sch import ibis.expr.types as ir -from ibis.backends.base import CanCreateDatabase, NoUrl -from ibis.backends.base.sqlglot import SQLGlotBackend +from ibis.backends import CanCreateDatabase, NoUrl from ibis.backends.flink.compiler import FlinkCompiler from ibis.backends.flink.ddl import ( CreateDatabase, @@ -22,6 +21,7 @@ InsertSelect, RenameTable, ) +from ibis.backends.sql import SQLBackend from ibis.backends.tests.errors import Py4JJavaError from ibis.expr.operations.udf import InputType from ibis.util import gen_name @@ -40,7 +40,7 @@ _INPUT_TYPE_TO_FUNC_TYPE = {InputType.PYTHON: "general", InputType.PANDAS: "pandas"} -class Backend(SQLGlotBackend, CanCreateDatabase, NoUrl): +class Backend(SQLBackend, CanCreateDatabase, NoUrl): name = "flink" compiler = FlinkCompiler() supports_temporary_tables = True diff --git a/ibis/backends/flink/compiler.py b/ibis/backends/flink/compiler.py index 6434a0ff5183..dda85eadb61c 100644 --- a/ibis/backends/flink/compiler.py +++ b/ibis/backends/flink/compiler.py @@ -8,10 +8,10 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import NULL, STAR, SQLGlotCompiler, paren -from ibis.backends.base.sqlglot.datatypes import FlinkType -from ibis.backends.base.sqlglot.dialects import Flink -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler, paren +from ibis.backends.sql.datatypes import FlinkType +from ibis.backends.sql.dialects import Flink +from ibis.backends.sql.rewrites import ( exclude_unsupported_window_frame_from_ops, exclude_unsupported_window_frame_from_rank, exclude_unsupported_window_frame_from_row_number, diff --git a/ibis/backends/flink/ddl.py b/ibis/backends/flink/ddl.py index b1cbd1da0db1..e12b45493d89 100644 --- a/ibis/backends/flink/ddl.py +++ b/ibis/backends/flink/ddl.py @@ -6,8 +6,8 @@ import ibis.common.exceptions as exc import ibis.expr.schema as sch -from ibis.backends.base.sqlglot.datatypes import FlinkType -from ibis.backends.base.sqlglot.ddl import DDL, DML, CreateDDL, DropObject +from ibis.backends.sql.datatypes import FlinkType +from ibis.backends.sql.ddl import DDL, DML, CreateDDL, DropObject from ibis.util import promote_list if TYPE_CHECKING: diff --git a/ibis/backends/impala/__init__.py b/ibis/backends/impala/__init__.py index 855bd65f7f3f..9719d3134fb7 100644 --- a/ibis/backends/impala/__init__.py +++ b/ibis/backends/impala/__init__.py @@ -20,7 +20,6 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base.sqlglot import SQLGlotBackend from ibis.backends.impala import ddl, udf from ibis.backends.impala.client import ImpalaTable from ibis.backends.impala.compiler import ImpalaCompiler @@ -41,6 +40,7 @@ wrap_uda, wrap_udf, ) +from ibis.backends.sql import SQLBackend from ibis.config import options if TYPE_CHECKING: @@ -62,7 +62,7 @@ ) -class Backend(SQLGlotBackend): +class Backend(SQLBackend): name = "impala" compiler = ImpalaCompiler() diff --git a/ibis/backends/impala/compiler.py b/ibis/backends/impala/compiler.py index f00dd14e18b4..e5831a152f00 100644 --- a/ibis/backends/impala/compiler.py +++ b/ibis/backends/impala/compiler.py @@ -7,10 +7,10 @@ import ibis.expr.datatypes as dt import ibis.expr.operations as ops from ibis import util -from ibis.backends.base.sqlglot.compiler import NULL, STAR, SQLGlotCompiler -from ibis.backends.base.sqlglot.datatypes import ImpalaType -from ibis.backends.base.sqlglot.dialects import Impala -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler +from ibis.backends.sql.datatypes import ImpalaType +from ibis.backends.sql.dialects import Impala +from ibis.backends.sql.rewrites import ( rewrite_empty_order_by_window, rewrite_first_to_first_value, rewrite_last_to_last_value, diff --git a/ibis/backends/impala/ddl.py b/ibis/backends/impala/ddl.py index 495ed48408eb..b97b71cebc58 100644 --- a/ibis/backends/impala/ddl.py +++ b/ibis/backends/impala/ddl.py @@ -5,8 +5,8 @@ import sqlglot as sg import ibis.expr.schema as sch -from ibis.backends.base.sqlglot.datatypes import ImpalaType -from ibis.backends.base.sqlglot.ddl import DDL, DML, CreateDDL, DropFunction, DropObject +from ibis.backends.sql.datatypes import ImpalaType +from ibis.backends.sql.ddl import DDL, DML, CreateDDL, DropFunction, DropObject class ImpalaBase: diff --git a/ibis/backends/mssql/__init__.py b/ibis/backends/mssql/__init__.py index bbedea3c0e3a..eb94f7f5c5a2 100644 --- a/ibis/backends/mssql/__init__.py +++ b/ibis/backends/mssql/__init__.py @@ -22,10 +22,10 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import CanCreateDatabase, CanCreateSchema, NoUrl -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import C +from ibis.backends import CanCreateDatabase, CanCreateSchema, NoUrl from ibis.backends.mssql.compiler import MSSQLCompiler +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import C if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping @@ -55,7 +55,7 @@ def datetimeoffset_to_datetime(value): ) -class Backend(SQLGlotBackend, CanCreateDatabase, CanCreateSchema, NoUrl): +class Backend(SQLBackend, CanCreateDatabase, CanCreateSchema, NoUrl): name = "mssql" compiler = MSSQLCompiler() supports_create_or_replace = False diff --git a/ibis/backends/mssql/compiler.py b/ibis/backends/mssql/compiler.py index 0a67d0c5b754..fac6c3f8c508 100644 --- a/ibis/backends/mssql/compiler.py +++ b/ibis/backends/mssql/compiler.py @@ -9,7 +9,7 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import ( +from ibis.backends.sql.compiler import ( FALSE, NULL, STAR, @@ -17,9 +17,9 @@ SQLGlotCompiler, paren, ) -from ibis.backends.base.sqlglot.datatypes import MSSQLType -from ibis.backends.base.sqlglot.dialects import MSSQL -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.datatypes import MSSQLType +from ibis.backends.sql.dialects import MSSQL +from ibis.backends.sql.rewrites import ( exclude_unsupported_window_frame_from_ops, exclude_unsupported_window_frame_from_row_number, p, diff --git a/ibis/backends/mysql/__init__.py b/ibis/backends/mysql/__init__.py index 6acfd5aa8585..50ba67708c4b 100644 --- a/ibis/backends/mysql/__init__.py +++ b/ibis/backends/mysql/__init__.py @@ -21,10 +21,10 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import CanCreateDatabase -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import TRUE, C +from ibis.backends import CanCreateDatabase from ibis.backends.mysql.compiler import MySQLCompiler +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import TRUE, C if TYPE_CHECKING: from collections.abc import Iterable, Mapping @@ -35,7 +35,7 @@ import ibis.expr.datatypes as dt -class Backend(SQLGlotBackend, CanCreateDatabase): +class Backend(SQLBackend, CanCreateDatabase): name = "mysql" compiler = MySQLCompiler() supports_create_or_replace = False diff --git a/ibis/backends/mysql/compiler.py b/ibis/backends/mysql/compiler.py index 41460616badd..a26df1405a58 100644 --- a/ibis/backends/mysql/compiler.py +++ b/ibis/backends/mysql/compiler.py @@ -10,10 +10,10 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import NULL, STAR, SQLGlotCompiler -from ibis.backends.base.sqlglot.datatypes import MySQLType -from ibis.backends.base.sqlglot.dialects import MySQL -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler +from ibis.backends.sql.datatypes import MySQLType +from ibis.backends.sql.dialects import MySQL +from ibis.backends.sql.rewrites import ( exclude_unsupported_window_frame_from_ops, exclude_unsupported_window_frame_from_rank, exclude_unsupported_window_frame_from_row_number, diff --git a/ibis/backends/oracle/__init__.py b/ibis/backends/oracle/__init__.py index fb6746bf3656..75a2743d8330 100644 --- a/ibis/backends/oracle/__init__.py +++ b/ibis/backends/oracle/__init__.py @@ -21,9 +21,9 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base.sqlglot import STAR, SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import TRUE, C from ibis.backends.oracle.compiler import OracleCompiler +from ibis.backends.sql import STAR, SQLBackend +from ibis.backends.sql.compiler import TRUE, C if TYPE_CHECKING: from collections.abc import Iterable @@ -32,7 +32,7 @@ import pyrrow as pa -class Backend(SQLGlotBackend): +class Backend(SQLBackend): name = "oracle" compiler = OracleCompiler() diff --git a/ibis/backends/oracle/compiler.py b/ibis/backends/oracle/compiler.py index 3c182d5c5e02..0934b97156fa 100644 --- a/ibis/backends/oracle/compiler.py +++ b/ibis/backends/oracle/compiler.py @@ -7,10 +7,10 @@ import ibis.common.exceptions as com import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import NULL, STAR, SQLGlotCompiler -from ibis.backends.base.sqlglot.datatypes import OracleType -from ibis.backends.base.sqlglot.dialects import Oracle -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler +from ibis.backends.sql.datatypes import OracleType +from ibis.backends.sql.dialects import Oracle +from ibis.backends.sql.rewrites import ( exclude_unsupported_window_frame_from_ops, exclude_unsupported_window_frame_from_row_number, replace_log2, diff --git a/ibis/backends/pandas/__init__.py b/ibis/backends/pandas/__init__.py index fa89b26af606..5cc23ed430f3 100644 --- a/ibis/backends/pandas/__init__.py +++ b/ibis/backends/pandas/__init__.py @@ -13,7 +13,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import BaseBackend, NoUrl +from ibis.backends import BaseBackend, NoUrl from ibis.formats.pandas import PandasData, PandasSchema from ibis.formats.pyarrow import PyArrowData diff --git a/ibis/backends/polars/__init__.py b/ibis/backends/polars/__init__.py index 694494275964..57ce60f18ed0 100644 --- a/ibis/backends/polars/__init__.py +++ b/ibis/backends/polars/__init__.py @@ -12,8 +12,7 @@ import ibis.expr.operations as ops import ibis.expr.schema as sch import ibis.expr.types as ir -from ibis.backends.base import BaseBackend, Database, NoUrl -from ibis.backends.base.sqlglot.dialects import Polars +from ibis.backends import BaseBackend, Database, NoUrl from ibis.backends.pandas.rewrites import ( bind_unbound_table, replace_parameter, @@ -21,6 +20,7 @@ ) from ibis.backends.polars.compiler import translate from ibis.backends.polars.datatypes import dtype_to_polars, schema_from_polars +from ibis.backends.sql.dialects import Polars from ibis.util import gen_name, normalize_filename if TYPE_CHECKING: diff --git a/ibis/backends/postgres/__init__.py b/ibis/backends/postgres/__init__.py index 3bf6c55e2a69..00004754f5de 100644 --- a/ibis/backends/postgres/__init__.py +++ b/ibis/backends/postgres/__init__.py @@ -24,9 +24,9 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import TRUE, C, ColGen, F from ibis.backends.postgres.compiler import PostgresCompiler +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import TRUE, C, ColGen, F from ibis.common.exceptions import InvalidDecoratorError if TYPE_CHECKING: @@ -42,7 +42,7 @@ def _verify_source_line(func_name: str, line: str): return line -class Backend(SQLGlotBackend): +class Backend(SQLBackend): name = "postgres" compiler = PostgresCompiler() supports_python_udfs = True diff --git a/ibis/backends/postgres/compiler.py b/ibis/backends/postgres/compiler.py index d6445fcdc308..1284962824a3 100644 --- a/ibis/backends/postgres/compiler.py +++ b/ibis/backends/postgres/compiler.py @@ -11,10 +11,10 @@ import ibis.expr.datatypes as dt import ibis.expr.operations as ops import ibis.expr.rules as rlz -from ibis.backends.base.sqlglot.compiler import NULL, STAR, SQLGlotCompiler, paren -from ibis.backends.base.sqlglot.datatypes import PostgresType -from ibis.backends.base.sqlglot.dialects import Postgres -from ibis.backends.base.sqlglot.rewrites import rewrite_sample_as_filter +from ibis.backends.sql.compiler import NULL, STAR, SQLGlotCompiler, paren +from ibis.backends.sql.datatypes import PostgresType +from ibis.backends.sql.dialects import Postgres +from ibis.backends.sql.rewrites import rewrite_sample_as_filter class PostgresUDFNode(ops.Value): diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index 1f50b9d42c38..955bdfab38b5 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -18,11 +18,11 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import CanCreateDatabase -from ibis.backends.base.sqlglot import SQLGlotBackend +from ibis.backends import CanCreateDatabase from ibis.backends.pyspark.compiler import PySparkCompiler from ibis.backends.pyspark.converter import PySparkPandasData from ibis.backends.pyspark.datatypes import PySparkSchema, PySparkType +from ibis.backends.sql import SQLBackend from ibis.expr.operations.udf import InputType from ibis.legacy.udf.vectorized import _coerce_to_series @@ -85,7 +85,7 @@ def __exit__(self, exc_type, exc_value, traceback): """No-op for compatibility.""" -class Backend(SQLGlotBackend, CanCreateDatabase): +class Backend(SQLBackend, CanCreateDatabase): name = "pyspark" compiler = PySparkCompiler() diff --git a/ibis/backends/pyspark/compiler.py b/ibis/backends/pyspark/compiler.py index cf84816e05c1..0167b57f0446 100644 --- a/ibis/backends/pyspark/compiler.py +++ b/ibis/backends/pyspark/compiler.py @@ -12,10 +12,10 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import FALSE, NULL, STAR, TRUE, SQLGlotCompiler -from ibis.backends.base.sqlglot.datatypes import PySparkType -from ibis.backends.base.sqlglot.dialects import PySpark -from ibis.backends.base.sqlglot.rewrites import p +from ibis.backends.sql.compiler import FALSE, NULL, STAR, TRUE, SQLGlotCompiler +from ibis.backends.sql.datatypes import PySparkType +from ibis.backends.sql.dialects import PySpark +from ibis.backends.sql.rewrites import p from ibis.common.patterns import replace from ibis.config import options from ibis.util import gen_name diff --git a/ibis/backends/risingwave/compiler.py b/ibis/backends/risingwave/compiler.py index f020c2089f33..df34cda256b0 100644 --- a/ibis/backends/risingwave/compiler.py +++ b/ibis/backends/risingwave/compiler.py @@ -7,10 +7,10 @@ import ibis.expr.datashape as ds import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import ALL_OPERATIONS -from ibis.backends.base.sqlglot.datatypes import RisingWaveType -from ibis.backends.base.sqlglot.dialects import RisingWave from ibis.backends.postgres.compiler import PostgresCompiler +from ibis.backends.sql.compiler import ALL_OPERATIONS +from ibis.backends.sql.datatypes import RisingWaveType +from ibis.backends.sql.dialects import RisingWave @public diff --git a/ibis/backends/snowflake/__init__.py b/ibis/backends/snowflake/__init__.py index bc38ebc19b56..a10c159e0d54 100644 --- a/ibis/backends/snowflake/__init__.py +++ b/ibis/backends/snowflake/__init__.py @@ -32,11 +32,11 @@ import ibis.expr.operations as ops import ibis.expr.types as ir from ibis import util -from ibis.backends.base import CanCreateDatabase, CanCreateSchema -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.datatypes import SnowflakeType +from ibis.backends import CanCreateDatabase, CanCreateSchema from ibis.backends.snowflake.compiler import SnowflakeCompiler from ibis.backends.snowflake.converter import SnowflakePandasData +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.datatypes import SnowflakeType if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping @@ -77,7 +77,7 @@ } -class Backend(SQLGlotBackend, CanCreateDatabase, CanCreateSchema): +class Backend(SQLBackend, CanCreateDatabase, CanCreateSchema): name = "snowflake" compiler = SnowflakeCompiler() supports_python_udfs = True diff --git a/ibis/backends/snowflake/compiler.py b/ibis/backends/snowflake/compiler.py index 56659499ae99..a1e96943583f 100644 --- a/ibis/backends/snowflake/compiler.py +++ b/ibis/backends/snowflake/compiler.py @@ -11,10 +11,10 @@ import ibis.expr.datatypes as dt import ibis.expr.operations as ops from ibis import util -from ibis.backends.base.sqlglot.compiler import NULL, C, FuncGen, SQLGlotCompiler -from ibis.backends.base.sqlglot.datatypes import SnowflakeType -from ibis.backends.base.sqlglot.dialects import Snowflake -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, C, FuncGen, SQLGlotCompiler +from ibis.backends.sql.datatypes import SnowflakeType +from ibis.backends.sql.dialects import Snowflake +from ibis.backends.sql.rewrites import ( exclude_unsupported_window_frame_from_ops, exclude_unsupported_window_frame_from_row_number, replace_log2, diff --git a/ibis/backends/snowflake/tests/conftest.py b/ibis/backends/snowflake/tests/conftest.py index 63b3497156bf..0742bfb53a13 100644 --- a/ibis/backends/snowflake/tests/conftest.py +++ b/ibis/backends/snowflake/tests/conftest.py @@ -16,13 +16,13 @@ import sqlglot as sg import ibis -from ibis.backends.base.sqlglot.datatypes import SnowflakeType from ibis.backends.conftest import TEST_TABLES +from ibis.backends.sql.datatypes import SnowflakeType from ibis.backends.tests.base import BackendTest from ibis.formats.pyarrow import PyArrowSchema if TYPE_CHECKING: - from ibis.backends.base import BaseBackend + from ibis.backends import BaseBackend def _get_url(): diff --git a/ibis/backends/snowflake/tests/test_datatypes.py b/ibis/backends/snowflake/tests/test_datatypes.py index c6e9ec17422a..7c0ad7d73c45 100644 --- a/ibis/backends/snowflake/tests/test_datatypes.py +++ b/ibis/backends/snowflake/tests/test_datatypes.py @@ -5,8 +5,8 @@ import ibis import ibis.expr.datatypes as dt -from ibis.backends.base.sqlglot.datatypes import SnowflakeType from ibis.backends.snowflake.tests.conftest import _get_url +from ibis.backends.sql.datatypes import SnowflakeType from ibis.util import gen_name dtypes = [ diff --git a/ibis/backends/base/sqlglot/__init__.py b/ibis/backends/sql/__init__.py similarity index 98% rename from ibis/backends/base/sqlglot/__init__.py rename to ibis/backends/sql/__init__.py index 247d6c971833..3d248014f738 100644 --- a/ibis/backends/base/sqlglot/__init__.py +++ b/ibis/backends/sql/__init__.py @@ -11,8 +11,8 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import BaseBackend -from ibis.backends.base.sqlglot.compiler import STAR +from ibis.backends import BaseBackend +from ibis.backends.sql.compiler import STAR if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping @@ -21,11 +21,11 @@ import pyarrow as pa import ibis.expr.datatypes as dt - from ibis.backends.base.sqlglot.compiler import SQLGlotCompiler + from ibis.backends.sql.compiler import SQLGlotCompiler from ibis.common.typing import SupportsSchema -class SQLGlotBackend(BaseBackend): +class SQLBackend(BaseBackend): compiler: ClassVar[SQLGlotCompiler] name: ClassVar[str] diff --git a/ibis/backends/base/sqlglot/compiler.py b/ibis/backends/sql/compiler.py similarity index 99% rename from ibis/backends/base/sqlglot/compiler.py rename to ibis/backends/sql/compiler.py index a6da4f6b71c5..14e121137e17 100644 --- a/ibis/backends/base/sqlglot/compiler.py +++ b/ibis/backends/sql/compiler.py @@ -18,7 +18,7 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.rewrites import ( add_one_to_nth_value_input, add_order_by_to_empty_ranking_window_functions, empty_in_values_right_side, @@ -34,7 +34,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir - from ibis.backends.base.sqlglot.datatypes import SqlglotType + from ibis.backends.sql.datatypes import SqlglotType def get_leaf_classes(op): diff --git a/ibis/backends/base/sqlglot/datatypes.py b/ibis/backends/sql/datatypes.py similarity index 100% rename from ibis/backends/base/sqlglot/datatypes.py rename to ibis/backends/sql/datatypes.py diff --git a/ibis/backends/base/sqlglot/ddl.py b/ibis/backends/sql/ddl.py similarity index 100% rename from ibis/backends/base/sqlglot/ddl.py rename to ibis/backends/sql/ddl.py diff --git a/ibis/backends/base/sqlglot/dialects.py b/ibis/backends/sql/dialects.py similarity index 100% rename from ibis/backends/base/sqlglot/dialects.py rename to ibis/backends/sql/dialects.py diff --git a/ibis/backends/base/sqlglot/rewrites.py b/ibis/backends/sql/rewrites.py similarity index 100% rename from ibis/backends/base/sqlglot/rewrites.py rename to ibis/backends/sql/rewrites.py diff --git a/ibis/backends/base/sqlglot/tests/__init__.py b/ibis/backends/sql/tests/__init__.py similarity index 100% rename from ibis/backends/base/sqlglot/tests/__init__.py rename to ibis/backends/sql/tests/__init__.py diff --git a/ibis/backends/base/sqlglot/tests/test_compiler.py b/ibis/backends/sql/tests/test_compiler.py similarity index 100% rename from ibis/backends/base/sqlglot/tests/test_compiler.py rename to ibis/backends/sql/tests/test_compiler.py diff --git a/ibis/backends/base/sqlglot/tests/test_datatypes.py b/ibis/backends/sql/tests/test_datatypes.py similarity index 96% rename from ibis/backends/base/sqlglot/tests/test_datatypes.py rename to ibis/backends/sql/tests/test_datatypes.py index 48abd4a969c5..07c50dbeb4f2 100644 --- a/ibis/backends/base/sqlglot/tests/test_datatypes.py +++ b/ibis/backends/sql/tests/test_datatypes.py @@ -8,7 +8,7 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.tests.strategies as its -from ibis.backends.base.sqlglot.datatypes import DuckDBType, PostgresType, SqlglotType +from ibis.backends.sql.datatypes import DuckDBType, PostgresType, SqlglotType def assert_dtype_roundtrip(ibis_type, sqlglot_expected=None): diff --git a/ibis/backends/sqlite/__init__.py b/ibis/backends/sqlite/__init__.py index b96753132ac7..9c70b66d7ed7 100644 --- a/ibis/backends/sqlite/__init__.py +++ b/ibis/backends/sqlite/__init__.py @@ -15,9 +15,9 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import UrlFromPath -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import C, F +from ibis.backends import UrlFromPath +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import C, F from ibis.backends.sqlite.compiler import SQLiteCompiler from ibis.backends.sqlite.converter import SQLitePandasData from ibis.backends.sqlite.udf import ignore_nulls, register_all @@ -42,7 +42,7 @@ def _quote(name: str) -> str: return sg.to_identifier(name, quoted=True).sql("sqlite") -class Backend(SQLGlotBackend, UrlFromPath): +class Backend(SQLBackend, UrlFromPath): name = "sqlite" compiler = SQLiteCompiler() supports_python_udfs = True diff --git a/ibis/backends/sqlite/compiler.py b/ibis/backends/sqlite/compiler.py index e543a680718e..bd92b175f79a 100644 --- a/ibis/backends/sqlite/compiler.py +++ b/ibis/backends/sqlite/compiler.py @@ -7,10 +7,10 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import NULL, SQLGlotCompiler -from ibis.backends.base.sqlglot.datatypes import SQLiteType -from ibis.backends.base.sqlglot.dialects import SQLite -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.compiler import NULL, SQLGlotCompiler +from ibis.backends.sql.datatypes import SQLiteType +from ibis.backends.sql.dialects import SQLite +from ibis.backends.sql.rewrites import ( rewrite_first_to_first_value, rewrite_last_to_last_value, rewrite_sample_as_filter, diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py index 1ebc504be679..813dfbb044d0 100644 --- a/ibis/backends/tests/test_client.py +++ b/ibis/backends/tests/test_client.py @@ -38,7 +38,7 @@ from ibis.util import gen_name if TYPE_CHECKING: - from ibis.backends.base import BaseBackend + from ibis.backends import BaseBackend @pytest.fixture diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py index 39c407beb959..0e46dde22368 100644 --- a/ibis/backends/tests/test_dot_sql.py +++ b/ibis/backends/tests/test_dot_sql.py @@ -10,10 +10,10 @@ import ibis import ibis.common.exceptions as com from ibis import _ -from ibis.backends.base import _get_backend_names +from ibis.backends import _get_backend_names # import here to load the dialect in to sqlglot so we can use it for transpilation -from ibis.backends.base.sqlglot.dialects import ( # noqa: F401 +from ibis.backends.sql.dialects import ( # noqa: F401 MSSQL, DataFusion, Druid, diff --git a/ibis/backends/tests/test_markers.py b/ibis/backends/tests/test_markers.py index c01662e9c181..16de77e9c6c8 100644 --- a/ibis/backends/tests/test_markers.py +++ b/ibis/backends/tests/test_markers.py @@ -2,7 +2,7 @@ import pytest -from ibis.backends.base import _get_backend_names +from ibis.backends import _get_backend_names all_backends = list(_get_backend_names()) diff --git a/ibis/backends/tests/test_temporal.py b/ibis/backends/tests/test_temporal.py index 2cf24eb95ed4..e4bf54e54dbf 100644 --- a/ibis/backends/tests/test_temporal.py +++ b/ibis/backends/tests/test_temporal.py @@ -15,7 +15,7 @@ import ibis import ibis.common.exceptions as com import ibis.expr.datatypes as dt -from ibis.backends.base import _get_backend_names +from ibis.backends import _get_backend_names from ibis.backends.conftest import is_older_than from ibis.backends.tests.errors import ( ArrowInvalid, diff --git a/ibis/backends/trino/__init__.py b/ibis/backends/trino/__init__.py index 6204976b5f25..8f1e39543c73 100644 --- a/ibis/backends/trino/__init__.py +++ b/ibis/backends/trino/__init__.py @@ -17,9 +17,9 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.backends.base import CanListDatabases, NoUrl -from ibis.backends.base.sqlglot import SQLGlotBackend -from ibis.backends.base.sqlglot.compiler import C +from ibis.backends import CanListDatabases, NoUrl +from ibis.backends.sql import SQLBackend +from ibis.backends.sql.compiler import C from ibis.backends.trino.compiler import TrinoCompiler if TYPE_CHECKING: @@ -31,7 +31,7 @@ import ibis.expr.operations as ops -class Backend(SQLGlotBackend, CanListDatabases, NoUrl): +class Backend(SQLBackend, CanListDatabases, NoUrl): name = "trino" compiler = TrinoCompiler() supports_create_or_replace = False diff --git a/ibis/backends/trino/compiler.py b/ibis/backends/trino/compiler.py index ec346284427f..e86e578a0d73 100644 --- a/ibis/backends/trino/compiler.py +++ b/ibis/backends/trino/compiler.py @@ -10,16 +10,16 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt import ibis.expr.operations as ops -from ibis.backends.base.sqlglot.compiler import ( +from ibis.backends.sql.compiler import ( FALSE, NULL, STAR, SQLGlotCompiler, paren, ) -from ibis.backends.base.sqlglot.datatypes import TrinoType -from ibis.backends.base.sqlglot.dialects import Trino -from ibis.backends.base.sqlglot.rewrites import ( +from ibis.backends.sql.datatypes import TrinoType +from ibis.backends.sql.dialects import Trino +from ibis.backends.sql.rewrites import ( exclude_unsupported_window_frame_from_ops, rewrite_first_to_first_value, rewrite_last_to_last_value, diff --git a/ibis/backends/trino/tests/test_datatypes.py b/ibis/backends/trino/tests/test_datatypes.py index fc7164fcfa87..5e3bc14a5403 100644 --- a/ibis/backends/trino/tests/test_datatypes.py +++ b/ibis/backends/trino/tests/test_datatypes.py @@ -4,7 +4,7 @@ from pytest import param import ibis.expr.datatypes as dt -from ibis.backends.base.sqlglot.datatypes import TrinoType +from ibis.backends.sql.datatypes import TrinoType dtypes = [ ("interval year to month", dt.Interval(unit="M")), diff --git a/ibis/config.py b/ibis/config.py index 50b82183c9f3..ecdb8a9aeef2 100644 --- a/ibis/config.py +++ b/ibis/config.py @@ -147,7 +147,7 @@ class Options(Config): A callable to use when logging. graphviz_repr : bool Render expressions as GraphViz PNGs when running in a Jupyter notebook. - default_backend : Optional[ibis.backends.base.BaseBackend], default None + default_backend : Optional[ibis.backends.BaseBackend], default None The default backend to use for execution, defaults to DuckDB if not set. context_adjustment : ContextAdjustment diff --git a/ibis/examples/__init__.py b/ibis/examples/__init__.py index f5cb5452a1b7..a55141f15242 100644 --- a/ibis/examples/__init__.py +++ b/ibis/examples/__init__.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: import ibis.expr.types as ir - from ibis.backends.base import BaseBackend + from ibis.backends import BaseBackend # These backends load the data directly using `read_csv`/`read_parquet`. All # other backends load the data using pyarrow, then passing it off to diff --git a/ibis/expr/api.py b/ibis/expr/api.py index 638ce9aaad99..5bdfb712d273 100644 --- a/ibis/expr/api.py +++ b/ibis/expr/api.py @@ -17,7 +17,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import selectors, util -from ibis.backends.base import BaseBackend, connect +from ibis.backends import BaseBackend, connect from ibis.common.deferred import Deferred, _, deferrable from ibis.common.dispatch import lazy_singledispatch from ibis.common.exceptions import IbisInputError diff --git a/ibis/expr/types/core.py b/ibis/expr/types/core.py index 10b27bdf20c9..36ad36b123fb 100644 --- a/ibis/expr/types/core.py +++ b/ibis/expr/types/core.py @@ -26,7 +26,7 @@ import torch import ibis.expr.types as ir - from ibis.backends.base import BaseBackend + from ibis.backends import BaseBackend TimeContext = tuple[pd.Timestamp, pd.Timestamp] diff --git a/ibis/streamlit/__init__.py b/ibis/streamlit/__init__.py index d95ea17160d3..d2b496244482 100644 --- a/ibis/streamlit/__init__.py +++ b/ibis/streamlit/__init__.py @@ -6,7 +6,7 @@ from streamlit.runtime.caching import cache_data import ibis -from ibis.backends.base import BaseBackend +from ibis.backends import BaseBackend __all__ = ["IbisConnection"] diff --git a/ibis/tests/benchmarks/test_benchmarks.py b/ibis/tests/benchmarks/test_benchmarks.py index c5e68a407f4c..343e1031e1e6 100644 --- a/ibis/tests/benchmarks/test_benchmarks.py +++ b/ibis/tests/benchmarks/test_benchmarks.py @@ -16,7 +16,7 @@ import ibis.expr.datatypes as dt import ibis.expr.operations as ops import ibis.expr.types as ir -from ibis.backends.base import _get_backend_names +from ibis.backends import _get_backend_names pytestmark = pytest.mark.benchmark @@ -676,7 +676,7 @@ def test_snowflake_medium_sized_to_pandas(benchmark): def test_parse_many_duckdb_types(benchmark): - from ibis.backends.base.sqlglot.datatypes import DuckDBType + from ibis.backends.sql.datatypes import DuckDBType def parse_many(types): list(map(DuckDBType.from_string, types)) diff --git a/ibis/tests/expr/mocks.py b/ibis/tests/expr/mocks.py index 8c6347428b21..8a4e81322750 100644 --- a/ibis/tests/expr/mocks.py +++ b/ibis/tests/expr/mocks.py @@ -20,7 +20,7 @@ import ibis.expr.operations as ops import ibis.expr.types as ir -from ibis.backends.base import BaseBackend +from ibis.backends import BaseBackend from ibis.expr.schema import Schema from ibis.expr.tests.conftest import MOCK_TABLES