Skip to content

Commit

Permalink
chore: add geospatial duckdb to new compiler
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud committed Dec 5, 2023
1 parent a14d7ba commit 6b50d7b
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 624 deletions.
57 changes: 32 additions & 25 deletions ibis/backends/duckdb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,9 @@
import ibis.expr.types as ir
from ibis import util
from ibis.backends.base import CanCreateSchema
from ibis.backends.base.sql import BaseBackend
from ibis.backends.base.sql.alchemy.geospatial import geospatial_supported
from ibis.backends.base.sqlglot import SQLGlotBackend
from ibis.backends.base.sqlglot.compiler import C, F, SQLGlotCompiler
from ibis.backends.base.sqlglot.compiler import STAR, C, F
from ibis.backends.base.sqlglot.datatypes import DuckDBType
from ibis.backends.duckdb.compiler import DuckDBCompiler
from ibis.backends.duckdb.datatypes import DuckDBPandasData
Expand Down Expand Up @@ -581,12 +580,16 @@ def read_json(
if not table_name:
table_name = util.gen_name("read_json")

options = [f"{key}={val}" for key, val in kwargs.items()]
options = [
sg.to_identifier(key).eq(sge.convert(val)) for key, val in kwargs.items()
]

self._create_temp_view(
table_name,
sg.select("*").from_(
sg.func("read_json_auto", normalize_filenames(source_list), *options)
sg.select(STAR).from_(
self.compiler.f.read_json_auto(
normalize_filenames(source_list), *options
)
),
)

Expand Down Expand Up @@ -636,13 +639,12 @@ def read_csv(
# We want to _usually_ quote arguments but if we quote `columns` it messes
# up DuckDB's struct parsing.
options = [
f'{key}="{val}"' if key != "columns" else f"{key}={val}"
for key, val in kwargs.items()
sg.to_identifier(key).eq(sge.convert(val)) for key, val in kwargs.items()
]

self._create_temp_view(
table_name,
sg.select("*").from_(sg.func("read_csv", source_list, *options)),
sg.select(STAR).from_(self.compiler.f.read_csv(source_list, *options)),
)

return self.table(table_name)
Expand Down Expand Up @@ -681,13 +683,21 @@ def read_geo(
# load geospatial extension
self.load_extension("spatial")

source_expr = sa.select(sa.literal_column("*")).select_from(
sa.func.st_read(util.normalize_filename(source), _format_kwargs(kwargs))
source_expr = sg.select(STAR).select_from(
self.compiler.f.st_read(
util.normalize_filename(source),
*(sg.to_identifier(key).eq(val) for key, val in kwargs.items()),
)
)

view = self._compile_temp_view(table_name, source_expr)
with self.begin() as con:
con.exec_driver_sql(view)
view = sge.Create(
kind="VIEW",
this=sg.table(table_name, quoted=self.compiler.quoted),
properties=sge.Properties(expressions=[sge.TemporalProperty()]),
expression=source_expr,
)
with self._safe_raw_sql(view):
pass
return self.table(table_name)

def read_parquet(
Expand Down Expand Up @@ -738,15 +748,12 @@ def _read_parquet_duckdb_native(
):
self._load_extensions(["httpfs"])

if kwargs:
options = [f"{key}={val}" for key, val in kwargs.items()]
pq_func = sg.func("read_parquet", source_list, *options)
else:
pq_func = sg.func("read_parquet", source_list)

options = [
sg.to_identifier(key).eq(sge.convert(val)) for key, val in kwargs.items()
]
self._create_temp_view(
table_name,
sg.select("*").from_(pq_func),
sg.select(STAR).from_(self.compiler.f.read_parquet(source_list, *options)),
)

def _read_parquet_pyarrow_dataset(
Expand Down Expand Up @@ -931,8 +938,8 @@ def read_postgres(

self._create_temp_view(
table_name,
sg.select("*").from_(
sg.func("postgres_scan_pushdown", uri, schema, table_name)
sg.select(STAR).from_(
self.compiler.f.postgres_scan_pushdown(uri, schema, table_name)
),
)

Expand Down Expand Up @@ -986,9 +993,9 @@ def read_sqlite(self, path: str | Path, table_name: str | None = None) -> ir.Tab

self._create_temp_view(
table_name,
sg.select("*").from_(
sg.func(
"sqlite_scan", sg.to_identifier(str(path), quoted=True), table_name
sg.select(STAR).from_(
self.compiler.f.sqlite_scan(
sg.to_identifier(str(path), quoted=True), table_name
)
),
)
Expand Down
30 changes: 30 additions & 0 deletions ibis/backends/duckdb/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,36 @@ def _neg_idx_to_pos(self, array, idx):
ops.TimeFromHMS: "make_time",
ops.TypeOf: "typeof",
ops.Unnest: "unnest",
ops.GeoPoint: "st_point",
ops.GeoAsText: "st_astext",
ops.GeoArea: "st_area",
ops.GeoBuffer: "st_buffer",
ops.GeoCentroid: "st_centroid",
ops.GeoContains: "st_contains",
ops.GeoCovers: "st_covers",
ops.GeoCoveredBy: "st_coveredby",
ops.GeoCrosses: "st_crosses",
ops.GeoDifference: "st_difference",
ops.GeoDisjoint: "st_disjoint",
ops.GeoDistance: "st_distance",
ops.GeoDWithin: "st_dwithin",
ops.GeoEndPoint: "st_endpoint",
ops.GeoEnvelope: "st_envelope",
ops.GeoEquals: "st_equals",
ops.GeoGeometryType: "st_geometrytype",
ops.GeoIntersection: "st_intersection",
ops.GeoIntersects: "st_intersects",
ops.GeoIsValid: "st_isvalid",
ops.GeoLength: "st_length",
ops.GeoNPoints: "st_npoints",
ops.GeoOverlaps: "st_overlaps",
ops.GeoStartPoint: "st_startpoint",
ops.GeoTouches: "st_touches",
ops.GeoUnion: "st_union",
ops.GeoUnaryUnion: "st_union_agg",
ops.GeoWithin: "st_within",
ops.GeoX: "st_x",
ops.GeoY: "st_y",
}


Expand Down
Loading

0 comments on commit 6b50d7b

Please sign in to comment.