now works with 1.1.0 + nightly fixes

voltrondata · Sep 12, 2024 · 102b20e · 102b20e
1 parent 61b5e2c
commit 102b20e
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 65 deletions.
diff --git a/src/backends/duckdb_backend.py b/src/backends/duckdb_backend.py
@@ -36,13 +36,11 @@ def create_connection(self):
             config={
                 "max_memory": "100GB",
                 "allow_unsigned_extensions": "true",
-                "autoinstall_extension_repository": "http://nightly-extensions.duckdb.org",
-                "custom_extension_repository": "http://nightly-extensions.duckdb.org",
                 "temp_directory": str(Path(".").resolve()),
             }
         )
-        self._connection.install_extension("substrait")
-        self._connection.load_extension("substrait")
+        self._connection.execute("FORCE INSTALL substrait FROM core_nightly")
+        self._connection.execute("LOAD substrait")
 
         return self._connection
 

diff --git a/src/gateway/tests/test_dataframe_api.py b/src/gateway/tests/test_dataframe_api.py
@@ -97,8 +97,6 @@ def mark_dataframe_tests_as_xfail(request):
         pytest.skip(reason="unionByName not supported in Substrait")
     if source != "spark" and originalname.startswith("test_exceptall"):
         pytest.skip(reason="exceptAll not supported in Substrait")
-    if source == "gateway-over-duckdb" and originalname in ["test_union", "test_unionall"]:
-        pytest.skip(reason="DuckDB treats all unions as distinct")
     if source == "gateway-over-datafusion" and originalname == "test_subtract":
         pytest.skip(reason="subtract not supported")
     if source == "gateway-over-datafusion" and originalname == "test_intersect":
@@ -130,24 +128,11 @@ def mark_dataframe_tests_as_xfail(request):
         pytest.skip(reason="argument count issue in DuckDB mapping")
     if source != "spark" and originalname in ["test_locate", "test_position"]:
         pytest.skip(reason="no direct Substrait analog")
-    if source == "gateway-over-duckdb" and originalname == "test_octet_length":
-        pytest.skip(reason="varchar octet_length not supported")
 
-    if source == "gateway-over-duckdb" and originalname == "test_sqrt":
-        pytest.skip(reason="behavior option ignored")
     if source != "spark" and originalname in ["test_rint", "test_bround"]:
         pytest.skip(reason="behavior option ignored")
     if source != "spark" and originalname in ["test_negative", "test_negate", "test_positive"]:
         pytest.skip(reason="custom implementation required")
-    if source == "gateway-over-duckdb" and originalname in [
-        "test_acosh",
-        "test_asinh",
-        "test_atanh",
-        "test_cosh",
-        "test_sinh",
-        "test_tanh",
-    ]:
-        pytest.skip(reason="missing implementation")
     if source == "gateway-over-datafusion" and originalname in ["test_sign", "test_signum"]:
         pytest.skip(reason="missing implementation")
     if source != "spark" and originalname in [
@@ -166,51 +151,6 @@ def mark_dataframe_tests_as_xfail(request):
     if source == "gateway-over-duckdb" and originalname == "test_row_number":
         pytest.skip(reason="window functions not yet implemented in DuckDB")
 
-    if source == "gateway-over-duckdb" and originalname in [
-        "test_filter",
-        "test_create_dataframe_and_temp_view",
-        "test_dropna",
-        "test_dropna_by_name",
-        "test_dropna_by_count",
-        "test_join",
-        "test_union_distinct",
-        "test_subtract",
-        "test_intersect",
-        "test_data_source_schema",
-        "test_data_source_filter",
-        "test_data_source_options",
-        "test_table_filter",
-        "test_broadcast",
-    ]:
-        pytest.skip(reason="unsupported root chain type")
-    if source == "gateway-over-duckdb" and originalname in [
-        "test_abs",
-        "test_ceil",
-        "test_ceiling",
-        "test_floor",
-        "test_round",
-        "test_toradians",
-        "test_radians",
-        "test_sign",
-        "test_signum",
-        "test_acos",
-        "test_asin",
-        "test_atan",
-        "test_atan2",
-        "test_cos",
-        "test_sin",
-        "test_tan",
-        "test_exp",
-        "test_pow",
-        "test_power",
-        "test_try_add",
-        "test_try_avg",
-        "test_try_divide",
-        "test_try_multiply",
-        "test_try_subtract",
-        "test_try_sum",
-    ]:
-        pytest.skip(reason="missing numbers table")
 
 # ruff: noqa: E712
 class TestDataFrameAPI:

diff --git a/src/gateway/tests/test_tpcds_sql.py b/src/gateway/tests/test_tpcds_sql.py
@@ -23,7 +23,6 @@ def mark_tests_as_xfail(request):
         if path.stem in ["01", "06", "10", "16", "30", "32", "35", "69", "81", "86", "92", "94"]:
             pytest.skip(reason="DuckDB needs Delim join")
         elif path.stem in [
-            "02",
             "03",
             "04",
             "05",