Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(polars): handle memtables like every other backend #10056

Merged
merged 1 commit into from
Sep 9, 2024
Merged

refactor(polars): handle memtables like every other backend #10056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ibis/backends/polars/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ def table(self, name: str) -> ir.Table:
schema = sch.infer(self._tables[name])
return ops.DatabaseTable(name, schema, self).to_expr()

def _register_in_memory_table(self, op: ops.InMemoryTable) -> None:
if (name := op.name) not in self._tables:
self._add_table(name, op.data.to_polars(op.schema).lazy())

@deprecated(
as_of="9.1",
instead="use the explicit `read_*` method for the filetype you are trying to read, e.g., read_parquet, read_csv, etc.",
Expand Down Expand Up @@ -466,6 +470,7 @@ def _to_dataframe(
streaming: bool = False,
**kwargs: Any,
) -> pl.DataFrame:
self._run_pre_execute_hooks(expr)
table_expr = expr.as_table()
lf = self.compile(table_expr, params=params, **kwargs)
if limit == "default":
Expand Down
8 changes: 6 additions & 2 deletions ibis/backends/polars/compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@
from math import isnan

import polars as pl
import sqlglot as sg

import ibis.common.exceptions as com
import ibis.expr.datatypes as dt
import ibis.expr.operations as ops
from ibis.backends.pandas.rewrites import PandasAsofJoin, PandasJoin, PandasRename
from ibis.backends.sql.compilers.base import STAR
from ibis.backends.sql.dialects import Polars
from ibis.expr.operations.udf import InputType
from ibis.formats.polars import PolarsType
from ibis.util import gen_name
Expand Down Expand Up @@ -64,8 +67,9 @@ def dummy_table(op, **kw):


@translate.register(ops.InMemoryTable)
def in_memory_table(op, **_):
return op.data.to_polars(op.schema).lazy()
def in_memory_table(op, *, ctx, **_):
sql = sg.select(STAR).from_(sg.to_identifier(op.name, quoted=True)).sql(Polars)
return ctx.execute(sql, eager=False)


def _make_duration(value, dtype):
Expand Down