From 79a54c6a4ffc48d663f0b366c6bb12f11fb93278 Mon Sep 17 00:00:00 2001 From: ncclementi Date: Mon, 9 Sep 2024 14:52:09 -0600 Subject: [PATCH 1/6] fix(datafusion): raise when attempting to create temp table chore: fix tests --- ibis/backends/datafusion/__init__.py | 7 +++--- .../datafusion/tests/test_register.py | 9 +++++++ ibis/backends/tests/test_client.py | 24 +++++++++++++++++-- ibis/backends/tests/test_string.py | 9 ++++++- 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index 915289864f34..55906b4b5f9e 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -622,10 +622,10 @@ def create_table( if schema is not None: schema = ibis.schema(schema) - properties = [] - if temp: - properties.append(sge.TemporaryProperty()) + raise NotImplementedError( + "Datafusion does not support temporary tables on sql mode" + ) quoted = self.compiler.quoted @@ -669,7 +669,6 @@ def create_table( create_stmt = sge.Create( kind="TABLE", this=target, - properties=sge.Properties(expressions=properties), expression=query, replace=overwrite, ) diff --git a/ibis/backends/datafusion/tests/test_register.py b/ibis/backends/datafusion/tests/test_register.py index 16a82973c7fa..3313f38adc6c 100644 --- a/ibis/backends/datafusion/tests/test_register.py +++ b/ibis/backends/datafusion/tests/test_register.py @@ -56,3 +56,12 @@ def test_create_table_with_uppercase_name(conn): tab = pa.table({"x": [1, 2, 3]}) conn.create_table("MY_TABLE", tab) assert conn.table("MY_TABLE").x.sum().execute() == 6 + + +def test_raise_create_temp_table(conn): + tab = pa.table({"x": [1, 2, 3]}) + with pytest.raises( + NotImplementedError, + match="Datafusion does not support temporary tables on sql mode", + ): + conn.create_table("my_temp_table", tab, temp=True) diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py index 2d80b56c4b4a..33f902cd1696 100644 --- a/ibis/backends/tests/test_client.py +++ b/ibis/backends/tests/test_client.py @@ -119,7 +119,14 @@ def test_create_table(backend, con, temp_table, func, sch): marks=[ pytest.mark.notyet(["clickhouse"], reason="Can't specify both"), pytest.mark.notyet( - ["pyspark", "trino", "exasol", "risingwave", "impala"], + [ + "pyspark", + "trino", + "exasol", + "risingwave", + "impala", + "datafusion", + ], reason="No support for temp tables", ), pytest.mark.notyet( @@ -145,7 +152,14 @@ def test_create_table(backend, con, temp_table, func, sch): id="temp, no overwrite", marks=[ pytest.mark.notyet( - ["pyspark", "trino", "exasol", "risingwave", "impala"], + [ + "pyspark", + "trino", + "exasol", + "risingwave", + "impala", + "datafusion", + ], reason="No support for temp tables", ), pytest.mark.notimpl(["mssql"], reason="Incorrect temp table syntax"), @@ -308,6 +322,9 @@ def test_create_table_from_schema(con, new_schema, temp_table): raises=com.IbisError, reason="`tbl_properties` is required when creating table with schema", ) +@pytest.mark.notimpl( + ["datafusion"], raises=NotImplementedError, reason="no temp table support via sql" +) def test_create_temporary_table_from_schema(con_no_data, new_schema): if con_no_data.name == "snowflake" and os.environ.get("SNOWFLAKE_SNOWPARK"): with pytest.raises( @@ -1562,6 +1579,9 @@ def test_json_to_pyarrow(con): assert result == expected +@pytest.mark.notimpl( + ["datafusion"], raises=NotImplementedError, reason="no temp table support via sql" +) @pytest.mark.notyet( ["risingwave", "exasol"], raises=com.UnsupportedOperationError, diff --git a/ibis/backends/tests/test_string.py b/ibis/backends/tests/test_string.py index 7f7c545dbed6..f1b74341ee35 100644 --- a/ibis/backends/tests/test_string.py +++ b/ibis/backends/tests/test_string.py @@ -1061,7 +1061,14 @@ def string_temp_table(backend, con): ) temp_table_name = gen_name("strings") - temp = backend.name() not in ["exasol", "impala", "pyspark", "risingwave", "trino"] + temp = backend.name() not in [ + "exasol", + "impala", + "pyspark", + "risingwave", + "trino", + "datafusion", + ] if backend.name() == "druid": yield "I HATE DRUID" else: From 2b2ca748ef28c4377894c3e48559216315a0a666 Mon Sep 17 00:00:00 2001 From: Naty Clementi Date: Tue, 10 Sep 2024 08:16:31 -0600 Subject: [PATCH 2/6] chore: update error message Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> --- ibis/backends/datafusion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index 55906b4b5f9e..8327ad67aa43 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -624,7 +624,7 @@ def create_table( if temp: raise NotImplementedError( - "Datafusion does not support temporary tables on sql mode" + "DataFusion does not support temporary tables" ) quoted = self.compiler.quoted From 406ddf9b7eaac5cf1a46bd176f2496663e5a9a6c Mon Sep 17 00:00:00 2001 From: Naty Clementi Date: Tue, 10 Sep 2024 08:16:43 -0600 Subject: [PATCH 3/6] chore: update error message Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> --- ibis/backends/datafusion/tests/test_register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis/backends/datafusion/tests/test_register.py b/ibis/backends/datafusion/tests/test_register.py index 3313f38adc6c..f50194c80ab5 100644 --- a/ibis/backends/datafusion/tests/test_register.py +++ b/ibis/backends/datafusion/tests/test_register.py @@ -62,6 +62,6 @@ def test_raise_create_temp_table(conn): tab = pa.table({"x": [1, 2, 3]}) with pytest.raises( NotImplementedError, - match="Datafusion does not support temporary tables on sql mode", + match="DataFusion does not support temporary tables", ): conn.create_table("my_temp_table", tab, temp=True) From 9fb34f9f7bbdcb373350efb25571fda79202b4e8 Mon Sep 17 00:00:00 2001 From: Naty Clementi Date: Tue, 10 Sep 2024 08:16:59 -0600 Subject: [PATCH 4/6] chore: update error message Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> --- ibis/backends/tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py index 33f902cd1696..4b3fd9cf4727 100644 --- a/ibis/backends/tests/test_client.py +++ b/ibis/backends/tests/test_client.py @@ -323,7 +323,7 @@ def test_create_table_from_schema(con, new_schema, temp_table): reason="`tbl_properties` is required when creating table with schema", ) @pytest.mark.notimpl( - ["datafusion"], raises=NotImplementedError, reason="no temp table support via sql" + ["datafusion"], raises=NotImplementedError, reason="no temp table support" ) def test_create_temporary_table_from_schema(con_no_data, new_schema): if con_no_data.name == "snowflake" and os.environ.get("SNOWFLAKE_SNOWPARK"): From c2466ffb15e2226787f8c25eb3c1c4e86cfcdfb9 Mon Sep 17 00:00:00 2001 From: Naty Clementi Date: Tue, 10 Sep 2024 08:17:12 -0600 Subject: [PATCH 5/6] chore: update error message Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> --- ibis/backends/tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py index 4b3fd9cf4727..5411eb2d05cb 100644 --- a/ibis/backends/tests/test_client.py +++ b/ibis/backends/tests/test_client.py @@ -1580,7 +1580,7 @@ def test_json_to_pyarrow(con): @pytest.mark.notimpl( - ["datafusion"], raises=NotImplementedError, reason="no temp table support via sql" + ["datafusion"], raises=NotImplementedError, reason="no temp table support" ) @pytest.mark.notyet( ["risingwave", "exasol"], From 7285da17f74beee622039fa91efa84ed054352d9 Mon Sep 17 00:00:00 2001 From: ncclementi Date: Tue, 10 Sep 2024 08:55:55 -0600 Subject: [PATCH 6/6] chore: fix linting --- ibis/backends/datafusion/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index 8327ad67aa43..6504f966399a 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -623,9 +623,7 @@ def create_table( schema = ibis.schema(schema) if temp: - raise NotImplementedError( - "DataFusion does not support temporary tables" - ) + raise NotImplementedError("DataFusion does not support temporary tables") quoted = self.compiler.quoted