fix(deps): update dependency datafusion to v42 (#10304)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Phillip Cloud <[email protected]>

ibis/backends/datafusion/__init__.py

@@ -166,7 +166,6 @@ def _get_schema_using_query(self, query: str) -> sch.Schema:
             this=table,
             kind="VIEW",
             expression=sg.parse_one(query, read=self.dialect),
-            properties=sge.Properties(expressions=[sge.TemporaryProperty()]),
         )
 
         with self._safe_raw_sql(src):
@@ -689,6 +688,9 @@ def truncate_table(
         with self._safe_raw_sql(sge.delete(ident)):
             pass
 
+    def _create_cached_table(self, name: str, expr: ir.Table) -> ir.Table:
+        return self.create_table(name, expr, schema=expr.schema())
+
 
 @contextlib.contextmanager
 def _create_and_drop_memtable(_conn, table_name, tmp_name, overwrite):

ibis/backends/tests/test_aggregation.py

@@ -1678,6 +1678,7 @@ def test_value_counts_on_expr(backend, alltypes, df):
     backend.assert_frame_equal(result, expected)
 
 
+@pytest.mark.xfail_version(datafusion=["datafusion==42"])
 def test_group_by_expr(backend, con):
     expr = (
         ibis.memtable(

ibis/backends/tests/test_client.py

@@ -294,11 +294,16 @@ def test_create_table_from_schema(con, new_schema, temp_table):
     reason="temporary tables not implemented",
     raises=NotImplementedError,
 )
-@pytest.mark.never(
+@pytest.mark.notyet(
     ["risingwave"],
     raises=com.UnsupportedOperationError,
     reason="Feature is not yet implemented: CREATE TEMPORARY TABLE",
 )
+@pytest.mark.notyet(
+    ["datafusion"],
+    raises=Exception,
+    reason="temp tables are not supported upstream in datafusion",
+)
 @pytest.mark.notimpl(
     ["flink"],
     raises=com.IbisError,

ibis/backends/tests/test_generic.py

@@ -1068,7 +1068,7 @@ def test_int_scalar(alltypes):
     assert isinstance(expr.execute(), int)
 
 
-@pytest.mark.notimpl(["datafusion", "polars", "druid"])
+@pytest.mark.notimpl(["polars", "druid"])
 @pytest.mark.notyet(
     ["clickhouse"], reason="https://github.com/ClickHouse/ClickHouse/issues/6697"
 )

ibis/backends/tests/test_map.py

@@ -555,7 +555,6 @@ def test_map_construct_dict(con, keys, values):
 )
 @mark_notimpl_risingwave_hstore
 @mark_notyet_postgres
-@mark_notyet_datafusion
 def test_map_construct_array_column(con, alltypes, df):
     expr = ibis.map(ibis.array([alltypes.string_col]), ibis.array([alltypes.int_col]))
     result = con.execute(expr)

ibis/backends/tests/tpc/ds/test_queries.py

@@ -1994,6 +1994,9 @@ def test_38(store_sales, catalog_sales, web_sales, date_dim, customer):
 
 
 @tpc_test("ds")
+@pytest.mark.notyet(
+    ["datafusion"], reason="incorrect results", raises=AssertionError, strict=False
+)
 def test_39(inventory, item, warehouse, date_dim):
     inv = (
         inventory.join(item, [("inv_item_sk", "i_item_sk")])
@@ -3980,10 +3983,6 @@ def test_74(customer, store_sales, date_dim, web_sales):
     return expr
 
 
-@pytest.mark.notyet(
-    ["datafusion"],
-    reason="Invalid argument error: RowConverter column schema mismatch, expected Int32 got Int64",
-)
 @tpc_test("ds")
 def test_75(
     catalog_sales,

pyproject.toml

@@ -55,7 +55,7 @@ clickhouse-connect = { version = ">=0.5.23,<1", optional = true, extras = [
   "numpy",
   "pandas",
 ] }
-datafusion = { version = ">=0.6,<42", optional = true }
+datafusion = { version = ">=0.6,<43", optional = true }
 db-dtypes = { version = ">=0.3,<2", optional = true }
 deltalake = { version = ">=0.9.0,<1", optional = true }
 duckdb = { version = ">=0.8.1,<1.2", optional = true }