diff --git a/ibis/backends/bigquery/compiler.py b/ibis/backends/bigquery/compiler.py index 5e06e5832842..eae9460d1104 100644 --- a/ibis/backends/bigquery/compiler.py +++ b/ibis/backends/bigquery/compiler.py @@ -111,8 +111,7 @@ class BigQueryCompiler(SQLGlotCompiler): ops.RegexReplace: "regexp_replace", ops.RegexSearch: "regexp_contains", ops.Time: "time", - ops.TimeFromHMS: "time", - ops.TimestampFromYMDHMS: "datetime", + ops.TimeFromHMS: "time_from_parts", ops.TimestampNow: "current_timestamp", ops.ExtractHost: "net.host", } @@ -271,16 +270,21 @@ def visti_StringFind(self, op, *, arg, substr, start, end): raise NotImplementedError("`end` not implemented for BigQuery string find") return self.f.strpos(arg, substr) + def visit_TimestampFromYMDHMS( + self, op, *, year, month, day, hours, minutes, seconds + ): + return self.f.anon.DATETIME(year, month, day, hours, minutes, seconds) + def visit_NonNullLiteral(self, op, *, value, dtype): if dtype.is_inet() or dtype.is_macaddr(): return sge.convert(str(value)) elif dtype.is_timestamp(): - funcname = "datetime" if dtype.timezone is None else "timestamp" - return self.f[funcname](value.isoformat()) + funcname = "DATETIME" if dtype.timezone is None else "TIMESTAMP" + return self.f.anon[funcname](value.isoformat()) elif dtype.is_date(): - return self.f.datefromparts(value.year, value.month, value.day) + return self.f.date_from_parts(value.year, value.month, value.day) elif dtype.is_time(): - return self.f.time(value.hour, value.minute, value.second) + return self.f.time_from_parts(value.hour, value.minute, value.second) elif dtype.is_binary(): return sge.Cast( this=sge.convert(value.hex()), diff --git a/ibis/backends/bigquery/tests/system/test_client.py b/ibis/backends/bigquery/tests/system/test_client.py index 048b8fb5cdef..dfb36bf69e61 100644 --- a/ibis/backends/bigquery/tests/system/test_client.py +++ b/ibis/backends/bigquery/tests/system/test_client.py @@ -115,22 +115,6 @@ def test_different_partition_col_name(monkeypatch, con): assert col in parted_alltypes.columns -def test_subquery_scalar_params(alltypes): - t = alltypes - p = ibis.param("timestamp").name("my_param") - expr = ( - t[["float_col", "timestamp_col", "int_col", "string_col"]][ - lambda t: t.timestamp_col < p - ] - .group_by("string_col") - .aggregate(foo=lambda t: t.float_col.sum()) - .foo.count() - .name("count") - ) - result = expr.compile(params={p: "20140101"}) - assert "datetime('2014-01-01T00:00:00')" in result - - def test_repr_struct_of_array_of_struct(): name = "foo" p = ibis.param("struct>>>").name(name) diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/datetime/index.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/datetime/index.sql index 2c01d43964b6..83162b1a4991 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/datetime/index.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/datetime/index.sql @@ -1,2 +1,2 @@ SELECT - MOD(EXTRACT(dayofweek FROM datetime('2017-01-01T04:55:59')) + 5, 7) AS `DayOfWeekIndex_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + MOD(EXTRACT(dayofweek FROM DATETIME('2017-01-01T04:55:59')) + 5, 7) AS `DayOfWeekIndex_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/datetime/name.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/datetime/name.sql index ed2e29f823ca..6cf0fc8cb6d2 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/datetime/name.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/datetime/name.sql @@ -1,2 +1,2 @@ SELECT - INITCAP(CAST(datetime('2017-01-01T04:55:59') AS STRING FORMAT 'DAY')) AS `DayOfWeekName_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + INITCAP(CAST(DATETIME('2017-01-01T04:55:59') AS STRING FORMAT 'DAY')) AS `DayOfWeekName_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/string_timestamp/index.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/string_timestamp/index.sql index 2c01d43964b6..83162b1a4991 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/string_timestamp/index.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/string_timestamp/index.sql @@ -1,2 +1,2 @@ SELECT - MOD(EXTRACT(dayofweek FROM datetime('2017-01-01T04:55:59')) + 5, 7) AS `DayOfWeekIndex_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + MOD(EXTRACT(dayofweek FROM DATETIME('2017-01-01T04:55:59')) + 5, 7) AS `DayOfWeekIndex_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/string_timestamp/name.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/string_timestamp/name.sql index ed2e29f823ca..6cf0fc8cb6d2 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/string_timestamp/name.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/string_timestamp/name.sql @@ -1,2 +1,2 @@ SELECT - INITCAP(CAST(datetime('2017-01-01T04:55:59') AS STRING FORMAT 'DAY')) AS `DayOfWeekName_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + INITCAP(CAST(DATETIME('2017-01-01T04:55:59') AS STRING FORMAT 'DAY')) AS `DayOfWeekName_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/timestamp/index.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/timestamp/index.sql index 2c01d43964b6..83162b1a4991 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/timestamp/index.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/timestamp/index.sql @@ -1,2 +1,2 @@ SELECT - MOD(EXTRACT(dayofweek FROM datetime('2017-01-01T04:55:59')) + 5, 7) AS `DayOfWeekIndex_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + MOD(EXTRACT(dayofweek FROM DATETIME('2017-01-01T04:55:59')) + 5, 7) AS `DayOfWeekIndex_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/timestamp/name.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/timestamp/name.sql index ed2e29f823ca..6cf0fc8cb6d2 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/timestamp/name.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_day_of_week/timestamp/name.sql @@ -1,2 +1,2 @@ SELECT - INITCAP(CAST(datetime('2017-01-01T04:55:59') AS STRING FORMAT 'DAY')) AS `DayOfWeekName_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + INITCAP(CAST(DATETIME('2017-01-01T04:55:59') AS STRING FORMAT 'DAY')) AS `DayOfWeekName_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_extract_temporal_from_timestamp/time/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_extract_temporal_from_timestamp/time/out.sql index f6b91dd9db01..798e1a7f807b 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_extract_temporal_from_timestamp/time/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_extract_temporal_from_timestamp/time/out.sql @@ -1,3 +1,3 @@ SELECT - time(`t0`.`ts`) AS `tmp` + TIME(`t0`.`ts`) AS `tmp` FROM `t` AS `t0` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/datetime/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/datetime/out.sql index ab23dad0560f..9bebaa6d9c6b 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/datetime/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/datetime/out.sql @@ -1,2 +1,2 @@ SELECT - EXTRACT(hour FROM datetime('2017-01-01T04:55:59')) AS `tmp` \ No newline at end of file + EXTRACT(hour FROM DATETIME('2017-01-01T04:55:59')) AS `tmp` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/string_time/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/string_time/out.sql index dcbea35ecd83..31bc04d8e7b6 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/string_time/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/string_time/out.sql @@ -1,2 +1,2 @@ SELECT - EXTRACT(hour FROM time(4, 55, 59)) AS `tmp` \ No newline at end of file + EXTRACT(hour FROM TIME(4, 55, 59)) AS `tmp` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/string_timestamp/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/string_timestamp/out.sql index ab23dad0560f..9bebaa6d9c6b 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/string_timestamp/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/string_timestamp/out.sql @@ -1,2 +1,2 @@ SELECT - EXTRACT(hour FROM datetime('2017-01-01T04:55:59')) AS `tmp` \ No newline at end of file + EXTRACT(hour FROM DATETIME('2017-01-01T04:55:59')) AS `tmp` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/time/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/time/out.sql index dcbea35ecd83..31bc04d8e7b6 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/time/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/time/out.sql @@ -1,2 +1,2 @@ SELECT - EXTRACT(hour FROM time(4, 55, 59)) AS `tmp` \ No newline at end of file + EXTRACT(hour FROM TIME(4, 55, 59)) AS `tmp` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/timestamp/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/timestamp/out.sql index ab23dad0560f..9bebaa6d9c6b 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/timestamp/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_timestamp_or_time/timestamp/out.sql @@ -1,2 +1,2 @@ SELECT - EXTRACT(hour FROM datetime('2017-01-01T04:55:59')) AS `tmp` \ No newline at end of file + EXTRACT(hour FROM DATETIME('2017-01-01T04:55:59')) AS `tmp` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/datetime/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/datetime/out.sql index f0e626ce7758..63d1d553ffd7 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/datetime/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/datetime/out.sql @@ -1,2 +1,2 @@ SELECT - EXTRACT(year FROM datetime('2017-01-01T04:55:59')) AS `ExtractYear_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + EXTRACT(year FROM DATETIME('2017-01-01T04:55:59')) AS `ExtractYear_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/string_timestamp/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/string_timestamp/out.sql index f0e626ce7758..63d1d553ffd7 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/string_timestamp/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/string_timestamp/out.sql @@ -1,2 +1,2 @@ SELECT - EXTRACT(year FROM datetime('2017-01-01T04:55:59')) AS `ExtractYear_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + EXTRACT(year FROM DATETIME('2017-01-01T04:55:59')) AS `ExtractYear_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/timestamp/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/timestamp/out.sql index f0e626ce7758..63d1d553ffd7 100644 --- a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/timestamp/out.sql +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_literal_year/timestamp/out.sql @@ -1,2 +1,2 @@ SELECT - EXTRACT(year FROM datetime('2017-01-01T04:55:59')) AS `ExtractYear_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file + EXTRACT(year FROM DATETIME('2017-01-01T04:55:59')) AS `ExtractYear_datetime_datetime_2017_1_1_4_55_59` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_subquery_scalar_params/out.sql b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_subquery_scalar_params/out.sql new file mode 100644 index 000000000000..ba7993c25320 --- /dev/null +++ b/ibis/backends/bigquery/tests/unit/snapshots/test_compiler/test_subquery_scalar_params/out.sql @@ -0,0 +1,16 @@ +SELECT + COUNT(`t2`.`foo`) AS `count` +FROM ( + SELECT + `t1`.`string_col`, + SUM(`t1`.`float_col`) AS `foo` + FROM ( + SELECT + * + FROM `alltypes` AS `t0` + WHERE + `t0`.`timestamp_col` < DATETIME('2014-01-01T00:00:00') + ) AS `t1` + GROUP BY + 1 +) AS `t2` \ No newline at end of file diff --git a/ibis/backends/bigquery/tests/unit/test_compiler.py b/ibis/backends/bigquery/tests/unit/test_compiler.py index bc231269ef06..9a1ec9dbc04e 100644 --- a/ibis/backends/bigquery/tests/unit/test_compiler.py +++ b/ibis/backends/bigquery/tests/unit/test_compiler.py @@ -646,3 +646,24 @@ def test_unnest(snapshot): ) def test_field_names_strip_whitespace(fieldname, expected): assert BigQueryCompiler._gen_valid_name(fieldname) == expected + + +def test_subquery_scalar_params(snapshot): + t = ibis.table( + schema={ + "float_col": "float64", + "timestamp_col": "timestamp", + "string_col": "string", + }, + name="alltypes", + ) + p = ibis.param("timestamp").name("my_param") + expr = ( + t.filter(lambda t: t.timestamp_col < p) + .group_by("string_col") + .aggregate(foo=lambda t: t.float_col.sum()) + .foo.count() + .name("count") + ) + result = ibis.to_sql(expr, params={p: "20140101"}, dialect="bigquery") + snapshot.assert_match(result, "out.sql") diff --git a/ibis/backends/sqlite/compiler.py b/ibis/backends/sqlite/compiler.py index 50cab3700728..805c7dfb5b47 100644 --- a/ibis/backends/sqlite/compiler.py +++ b/ibis/backends/sqlite/compiler.py @@ -312,7 +312,7 @@ def visit_DateTruncate(self, op, *, arg, unit): return self._temporal_truncate(self.f.date, arg, unit) def visit_TimestampTruncate(self, op, *, arg, unit): - return self._temporal_truncate(self.f.datetime, arg, unit) + return self._temporal_truncate(self.f.anon.datetime, arg, unit) def visit_DateArithmetic(self, op, *, left, right): unit = op.right.dtype.unit diff --git a/ibis/backends/tests/snapshots/test_generic/test_many_subqueries/oracle/out.sql b/ibis/backends/tests/snapshots/test_generic/test_many_subqueries/oracle/out.sql index d92a17b9526e..a1280f31a6e4 100644 --- a/ibis/backends/tests/snapshots/test_generic/test_many_subqueries/oracle/out.sql +++ b/ibis/backends/tests/snapshots/test_generic/test_many_subqueries/oracle/out.sql @@ -1,12 +1,12 @@ WITH "t1" AS ( SELECT "t0"."street", - ROW_NUMBER() OVER (ORDER BY "t0"."street" ASC NULLS LAST) - 1 AS "key" + ROW_NUMBER() OVER (ORDER BY "t0"."street" ASC) - 1 AS "key" FROM "data" "t0" ), "t7" AS ( SELECT "t6"."street", - ROW_NUMBER() OVER (ORDER BY "t6"."street" ASC NULLS LAST) - 1 AS "key" + ROW_NUMBER() OVER (ORDER BY "t6"."street" ASC) - 1 AS "key" FROM ( SELECT "t3"."street", diff --git a/poetry.lock b/poetry.lock index cdfb2dca9c11..d3ff275ee488 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6756,18 +6756,18 @@ sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlglot" -version = "25.2.0" +version = "25.3.0" description = "An easily customizable SQL parser and transpiler" optional = false python-versions = ">=3.7" files = [ - {file = "sqlglot-25.2.0-py3-none-any.whl", hash = "sha256:734dedf43c1a453eb0ab02dd9e245290855bb44ee3a569acb661fd378484955f"}, - {file = "sqlglot-25.2.0.tar.gz", hash = "sha256:545ee36ab9f94bb227715d6bff63872509280c584046854d4d977daff169c6e2"}, + {file = "sqlglot-25.3.0-py3-none-any.whl", hash = "sha256:1cca732e7c2ba4fe86665d8e05d7af49b3f2a5e7ec45a8f3ab0649d4207351b9"}, + {file = "sqlglot-25.3.0.tar.gz", hash = "sha256:c4ce5e38148c29f3bb19d8dcf1bfaaa4833eb283e9e6771e39a7e90dd6b79a0c"}, ] [package.extras] dev = ["duckdb (>=0.6)", "maturin (>=1.4,<2.0)", "mypy", "pandas", "pandas-stubs", "pdoc", "pre-commit", "python-dateutil", "ruff (==0.4.3)", "types-python-dateutil", "typing-extensions"] -rs = ["sqlglotrs (==0.2.6)"] +rs = ["sqlglotrs (==0.2.7)"] [[package]] name = "stack-data" @@ -7712,4 +7712,4 @@ visualization = ["graphviz"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "9591349c5bcf0b90e91ccab81fd4e661d29fe5ee9544e31ae599f2ec6b9e5893" +content-hash = "a0ce04c4268df6dda6a10db2ea858ad3324d933f7799ad3e16102a9dde9a230f" diff --git a/pyproject.toml b/pyproject.toml index 448932780bde..7c02c4dea749 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ pyarrow-hotfix = ">=0.4,<1" python-dateutil = ">=2.8.2,<3" pytz = ">=2022.7" rich = ">=12.4.4,<14" -sqlglot = ">=23.4,<25.3" +sqlglot = ">=23.4,<25.4" toolz = ">=0.11,<1" typing-extensions = ">=4.3.0,<5" black = { version = ">=22.1.0,<25", optional = true } diff --git a/requirements-dev.txt b/requirements-dev.txt index 983ac0d8d1f7..d09c1548dd6e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -256,7 +256,7 @@ sortedcontainers==2.4.0 ; python_version >= "3.10" and python_version < "4.0" soupsieve==2.5 ; python_version >= "3.10" and python_version < "3.13" sphobjinv==2.3.1.1 ; python_version >= "3.10" and python_version < "3.13" sqlalchemy==2.0.31 ; python_version >= "3.10" and python_version < "3.13" -sqlglot==25.2.0 ; python_version >= "3.10" and python_version < "4.0" +sqlglot==25.3.0 ; python_version >= "3.10" and python_version < "4.0" stack-data==0.6.3 ; python_version >= "3.10" and python_version < "4.0" statsmodels==0.14.2 ; python_version >= "3.10" and python_version < "3.13" stdlib-list==0.10.0 ; python_version >= "3.10" and python_version < "4.0"