From 7f08bbc4ba70a6f85f0297847dd36a89320f8771 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:23:12 -0400 Subject: [PATCH 1/4] test(snowflake): fix snapshots --- .../test_h20/test_tpc_h20/snowflake/h20.sql | 184 ++++++++++-------- .../test_h22/test_tpc_h22/snowflake/h22.sql | 115 ++++++----- 2 files changed, 166 insertions(+), 133 deletions(-) diff --git a/ibis/backends/tests/tpch/snapshots/test_h20/test_tpc_h20/snowflake/h20.sql b/ibis/backends/tests/tpch/snapshots/test_h20/test_tpc_h20/snowflake/h20.sql index ee7298e9314c..cc3df47afbcd 100644 --- a/ibis/backends/tests/tpch/snapshots/test_h20/test_tpc_h20/snowflake/h20.sql +++ b/ibis/backends/tests/tpch/snapshots/test_h20/test_tpc_h20/snowflake/h20.sql @@ -1,93 +1,115 @@ SELECT - "t12"."s_name", - "t12"."s_address" + "t16"."s_name", + "t16"."s_address" FROM ( SELECT - "t8"."s_suppkey", - "t8"."s_name", - "t8"."s_address", - "t8"."s_nationkey", - "t8"."s_phone", - "t8"."s_acctbal", - "t8"."s_comment", - "t9"."n_nationkey", - "t9"."n_name", - "t9"."n_regionkey", - "t9"."n_comment" + "t12"."s_suppkey", + "t12"."s_name", + "t12"."s_address", + "t12"."s_nationkey", + "t12"."s_phone", + "t12"."s_acctbal", + "t12"."s_comment", + "t12"."n_nationkey", + "t12"."n_name", + "t12"."n_regionkey", + "t12"."n_comment" FROM ( SELECT - "t0"."S_SUPPKEY" AS "s_suppkey", - "t0"."S_NAME" AS "s_name", - "t0"."S_ADDRESS" AS "s_address", - "t0"."S_NATIONKEY" AS "s_nationkey", - "t0"."S_PHONE" AS "s_phone", - "t0"."S_ACCTBAL" AS "s_acctbal", - "t0"."S_COMMENT" AS "s_comment" - FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."SUPPLIER" AS "t0" - ) AS "t8" - INNER JOIN ( - SELECT - "t2"."N_NATIONKEY" AS "n_nationkey", - "t2"."N_NAME" AS "n_name", - "t2"."N_REGIONKEY" AS "n_regionkey", - "t2"."N_COMMENT" AS "n_comment" - FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."NATION" AS "t2" - ) AS "t9" - ON "t8"."s_nationkey" = "t9"."n_nationkey" -) AS "t12" -WHERE - "t12"."n_name" = 'CANADA' - AND "t12"."s_suppkey" IN ( - SELECT - "t6"."ps_suppkey" + "t8"."s_suppkey", + "t8"."s_name", + "t8"."s_address", + "t8"."s_nationkey", + "t8"."s_phone", + "t8"."s_acctbal", + "t8"."s_comment", + "t9"."n_nationkey", + "t9"."n_name", + "t9"."n_regionkey", + "t9"."n_comment" FROM ( SELECT - "t1"."PS_PARTKEY" AS "ps_partkey", - "t1"."PS_SUPPKEY" AS "ps_suppkey", - "t1"."PS_AVAILQTY" AS "ps_availqty", - "t1"."PS_SUPPLYCOST" AS "ps_supplycost", - "t1"."PS_COMMENT" AS "ps_comment" - FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."PARTSUPP" AS "t1" - ) AS "t6" - WHERE - "t6"."ps_partkey" IN ( + "t0"."S_SUPPKEY" AS "s_suppkey", + "t0"."S_NAME" AS "s_name", + "t0"."S_ADDRESS" AS "s_address", + "t0"."S_NATIONKEY" AS "s_nationkey", + "t0"."S_PHONE" AS "s_phone", + "t0"."S_ACCTBAL" AS "s_acctbal", + "t0"."S_COMMENT" AS "s_comment" + FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."SUPPLIER" AS "t0" + ) AS "t8" + INNER JOIN ( + SELECT + "t1"."N_NATIONKEY" AS "n_nationkey", + "t1"."N_NAME" AS "n_name", + "t1"."N_REGIONKEY" AS "n_regionkey", + "t1"."N_COMMENT" AS "n_comment" + FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."NATION" AS "t1" + ) AS "t9" + ON "t8"."s_nationkey" = "t9"."n_nationkey" + ) AS "t12" + WHERE + "t12"."n_name" = 'CANADA' + AND "t12"."s_suppkey" IN ( + SELECT + "t14"."ps_suppkey" + FROM ( SELECT - "t3"."P_PARTKEY" AS "p_partkey" - FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."PART" AS "t3" - WHERE - "t3"."P_NAME" LIKE 'forest%' - ) - AND "t6"."ps_availqty" > ( - ( + "t7"."ps_partkey", + "t7"."ps_suppkey", + "t7"."ps_availqty", + "t7"."ps_supplycost", + "t7"."ps_comment" + FROM ( SELECT - SUM("t11"."l_quantity") AS "Sum(l_quantity)" - FROM ( + "t2"."PS_PARTKEY" AS "ps_partkey", + "t2"."PS_SUPPKEY" AS "ps_suppkey", + "t2"."PS_AVAILQTY" AS "ps_availqty", + "t2"."PS_SUPPLYCOST" AS "ps_supplycost", + "t2"."PS_COMMENT" AS "ps_comment" + FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."PARTSUPP" AS "t2" + ) AS "t7" + WHERE + "t7"."ps_partkey" IN ( SELECT - "t4"."L_ORDERKEY" AS "l_orderkey", - "t4"."L_PARTKEY" AS "l_partkey", - "t4"."L_SUPPKEY" AS "l_suppkey", - "t4"."L_LINENUMBER" AS "l_linenumber", - "t4"."L_QUANTITY" AS "l_quantity", - "t4"."L_EXTENDEDPRICE" AS "l_extendedprice", - "t4"."L_DISCOUNT" AS "l_discount", - "t4"."L_TAX" AS "l_tax", - "t4"."L_RETURNFLAG" AS "l_returnflag", - "t4"."L_LINESTATUS" AS "l_linestatus", - "t4"."L_SHIPDATE" AS "l_shipdate", - "t4"."L_COMMITDATE" AS "l_commitdate", - "t4"."L_RECEIPTDATE" AS "l_receiptdate", - "t4"."L_SHIPINSTRUCT" AS "l_shipinstruct", - "t4"."L_SHIPMODE" AS "l_shipmode", - "t4"."L_COMMENT" AS "l_comment" - FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."LINEITEM" AS "t4" + "t3"."P_PARTKEY" AS "p_partkey" + FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."PART" AS "t3" WHERE - "t4"."L_PARTKEY" = "t6"."ps_partkey" - AND "t4"."L_SUPPKEY" = "t6"."ps_suppkey" - AND "t4"."L_SHIPDATE" >= DATE_FROM_PARTS(1994, 1, 1) - AND "t4"."L_SHIPDATE" < DATE_FROM_PARTS(1995, 1, 1) - ) AS "t11" - ) * 0.5 - ) - ) + "t3"."P_NAME" LIKE 'forest%' + ) + AND "t7"."ps_availqty" > ( + ( + SELECT + SUM("t11"."l_quantity") AS "Sum(l_quantity)" + FROM ( + SELECT + "t4"."L_ORDERKEY" AS "l_orderkey", + "t4"."L_PARTKEY" AS "l_partkey", + "t4"."L_SUPPKEY" AS "l_suppkey", + "t4"."L_LINENUMBER" AS "l_linenumber", + "t4"."L_QUANTITY" AS "l_quantity", + "t4"."L_EXTENDEDPRICE" AS "l_extendedprice", + "t4"."L_DISCOUNT" AS "l_discount", + "t4"."L_TAX" AS "l_tax", + "t4"."L_RETURNFLAG" AS "l_returnflag", + "t4"."L_LINESTATUS" AS "l_linestatus", + "t4"."L_SHIPDATE" AS "l_shipdate", + "t4"."L_COMMITDATE" AS "l_commitdate", + "t4"."L_RECEIPTDATE" AS "l_receiptdate", + "t4"."L_SHIPINSTRUCT" AS "l_shipinstruct", + "t4"."L_SHIPMODE" AS "l_shipmode", + "t4"."L_COMMENT" AS "l_comment" + FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."LINEITEM" AS "t4" + WHERE + "t4"."L_PARTKEY" = "t7"."ps_partkey" + AND "t4"."L_SUPPKEY" = "t7"."ps_suppkey" + AND "t4"."L_SHIPDATE" >= DATE_FROM_PARTS(1994, 1, 1) + AND "t4"."L_SHIPDATE" < DATE_FROM_PARTS(1995, 1, 1) + ) AS "t11" + ) * 0.5 + ) + ) AS "t14" + ) +) AS "t16" ORDER BY - "t12"."s_name" ASC \ No newline at end of file + "t16"."s_name" ASC \ No newline at end of file diff --git a/ibis/backends/tests/tpch/snapshots/test_h22/test_tpc_h22/snowflake/h22.sql b/ibis/backends/tests/tpch/snapshots/test_h22/test_tpc_h22/snowflake/h22.sql index e2c6887c4b7d..7b3a0aa5a09a 100644 --- a/ibis/backends/tests/tpch/snapshots/test_h22/test_tpc_h22/snowflake/h22.sql +++ b/ibis/backends/tests/tpch/snapshots/test_h22/test_tpc_h22/snowflake/h22.sql @@ -1,67 +1,78 @@ SELECT - "t7"."cntrycode", - "t7"."numcust", - "t7"."totacctbal" + "t8"."cntrycode", + "t8"."numcust", + "t8"."totacctbal" FROM ( SELECT - "t6"."cntrycode", + "t7"."cntrycode", COUNT(*) AS "numcust", - SUM("t6"."c_acctbal") AS "totacctbal" + SUM("t7"."c_acctbal") AS "totacctbal" FROM ( SELECT - SUBSTRING("t2"."c_phone", IFF(( + SUBSTRING("t6"."c_phone", IFF(( 0 + 1 - ) >= 1, 0 + 1, 0 + 1 + LENGTH("t2"."c_phone")), 2) AS "cntrycode", - "t2"."c_acctbal" + ) >= 1, 0 + 1, 0 + 1 + LENGTH("t6"."c_phone")), 2) AS "cntrycode", + "t6"."c_acctbal" FROM ( SELECT - "t0"."C_CUSTKEY" AS "c_custkey", - "t0"."C_NAME" AS "c_name", - "t0"."C_ADDRESS" AS "c_address", - "t0"."C_NATIONKEY" AS "c_nationkey", - "t0"."C_PHONE" AS "c_phone", - "t0"."C_ACCTBAL" AS "c_acctbal", - "t0"."C_MKTSEGMENT" AS "c_mktsegment", - "t0"."C_COMMENT" AS "c_comment" - FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."CUSTOMER" AS "t0" - ) AS "t2" - WHERE - SUBSTRING("t2"."c_phone", IFF(( - 0 + 1 - ) >= 1, 0 + 1, 0 + 1 + LENGTH("t2"."c_phone")), 2) IN ('13', '31', '23', '29', '30', '18', '17') - AND "t2"."c_acctbal" > ( + "t2"."c_custkey", + "t2"."c_name", + "t2"."c_address", + "t2"."c_nationkey", + "t2"."c_phone", + "t2"."c_acctbal", + "t2"."c_mktsegment", + "t2"."c_comment" + FROM ( SELECT - AVG("t3"."c_acctbal") AS "Mean(c_acctbal)" - FROM ( - SELECT - "t0"."C_CUSTKEY" AS "c_custkey", - "t0"."C_NAME" AS "c_name", - "t0"."C_ADDRESS" AS "c_address", - "t0"."C_NATIONKEY" AS "c_nationkey", - "t0"."C_PHONE" AS "c_phone", - "t0"."C_ACCTBAL" AS "c_acctbal", - "t0"."C_MKTSEGMENT" AS "c_mktsegment", - "t0"."C_COMMENT" AS "c_comment" - FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."CUSTOMER" AS "t0" - WHERE - "t0"."C_ACCTBAL" > 0.0 - AND SUBSTRING("t0"."C_PHONE", IFF(( - 0 + 1 - ) >= 1, 0 + 1, 0 + 1 + LENGTH("t0"."C_PHONE")), 2) IN ('13', '31', '23', '29', '30', '18', '17') - ) AS "t3" - ) - AND NOT ( - EXISTS( + "t0"."C_CUSTKEY" AS "c_custkey", + "t0"."C_NAME" AS "c_name", + "t0"."C_ADDRESS" AS "c_address", + "t0"."C_NATIONKEY" AS "c_nationkey", + "t0"."C_PHONE" AS "c_phone", + "t0"."C_ACCTBAL" AS "c_acctbal", + "t0"."C_MKTSEGMENT" AS "c_mktsegment", + "t0"."C_COMMENT" AS "c_comment" + FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."CUSTOMER" AS "t0" + ) AS "t2" + WHERE + SUBSTRING("t2"."c_phone", IFF(( + 0 + 1 + ) >= 1, 0 + 1, 0 + 1 + LENGTH("t2"."c_phone")), 2) IN ('13', '31', '23', '29', '30', '18', '17') + AND "t2"."c_acctbal" > ( SELECT - 1 - FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."ORDERS" AS "t1" - WHERE - "t1"."O_CUSTKEY" = "t2"."c_custkey" + AVG("t3"."c_acctbal") AS "Mean(c_acctbal)" + FROM ( + SELECT + "t0"."C_CUSTKEY" AS "c_custkey", + "t0"."C_NAME" AS "c_name", + "t0"."C_ADDRESS" AS "c_address", + "t0"."C_NATIONKEY" AS "c_nationkey", + "t0"."C_PHONE" AS "c_phone", + "t0"."C_ACCTBAL" AS "c_acctbal", + "t0"."C_MKTSEGMENT" AS "c_mktsegment", + "t0"."C_COMMENT" AS "c_comment" + FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."CUSTOMER" AS "t0" + WHERE + "t0"."C_ACCTBAL" > 0.0 + AND SUBSTRING("t0"."C_PHONE", IFF(( + 0 + 1 + ) >= 1, 0 + 1, 0 + 1 + LENGTH("t0"."C_PHONE")), 2) IN ('13', '31', '23', '29', '30', '18', '17') + ) AS "t3" + ) + AND NOT ( + EXISTS( + SELECT + 1 + FROM "SNOWFLAKE_SAMPLE_DATA"."TPCH_SF1"."ORDERS" AS "t1" + WHERE + "t1"."O_CUSTKEY" = "t2"."c_custkey" + ) ) - ) - ) AS "t6" + ) AS "t6" + ) AS "t7" GROUP BY 1 -) AS "t7" +) AS "t8" ORDER BY - "t7"."cntrycode" ASC \ No newline at end of file + "t8"."cntrycode" ASC \ No newline at end of file From 5185fe9c6cd7ec71ac3a535ac335a7c780a4d1af Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 10 Apr 2024 10:30:24 -0400 Subject: [PATCH 2/4] disable cache for nwo --- .github/workflows/ibis-backends.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml index 45a22b28aac7..edc2810b83ce 100644 --- a/.github/workflows/ibis-backends.yml +++ b/.github/workflows/ibis-backends.yml @@ -398,7 +398,6 @@ jobs: id: install_python with: python-version: ${{ matrix.python-version }} - cache: poetry - name: install ibis run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }}" From da1e5e6a5c4a89a2fd308acf7e4b7d86bd0fd3b3 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 10 Apr 2024 10:39:08 -0400 Subject: [PATCH 3/4] ci(deps): remove dependency caching except when installing all extras --- .github/workflows/ibis-backends-cloud.yml | 1 - .github/workflows/ibis-backends.yml | 1 - .github/workflows/ibis-main.yml | 4 +--- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ibis-backends-cloud.yml b/.github/workflows/ibis-backends-cloud.yml index 127e42540f94..cbca254eedab 100644 --- a/.github/workflows/ibis-backends-cloud.yml +++ b/.github/workflows/ibis-backends-cloud.yml @@ -94,7 +94,6 @@ jobs: id: install_python with: python-version: ${{ matrix.python-version }} - cache: poetry - name: install ibis run: poetry install --without dev --without docs --extras "${{ join(matrix.backend.extras, ' ') }}" diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml index edc2810b83ce..e10009bcc751 100644 --- a/.github/workflows/ibis-backends.yml +++ b/.github/workflows/ibis-backends.yml @@ -66,7 +66,6 @@ jobs: id: install_python with: python-version: ${{ matrix.python-version }} - cache: poetry - name: install ibis run: poetry install --without dev --without docs --extras bigquery diff --git a/.github/workflows/ibis-main.yml b/.github/workflows/ibis-main.yml index dfadf5232498..9c72c15d680e 100644 --- a/.github/workflows/ibis-main.yml +++ b/.github/workflows/ibis-main.yml @@ -62,7 +62,6 @@ jobs: id: install_python with: python-version: ${{ matrix.python-version }} - cache: poetry - name: install ${{ matrix.os }} system dependencies if: matrix.os == 'ubuntu-latest' @@ -114,7 +113,6 @@ jobs: id: install_python with: python-version: "3.12" - cache: poetry - name: install system dependencies run: | @@ -157,7 +155,7 @@ jobs: cache: poetry - name: install ibis with all extras - run: poetry install --without dev --without docs --extras all + run: poetry install --without dev --without docs --all-extras - uses: extractions/setup-just@v2 env: From afd0cc37acd9868dd27aabd901a89bc0bce39f33 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Wed, 10 Apr 2024 15:07:28 -0400 Subject: [PATCH 4/4] ci: wat --- .github/workflows/ibis-backends.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml index e10009bcc751..2674a9d51e9a 100644 --- a/.github/workflows/ibis-backends.yml +++ b/.github/workflows/ibis-backends.yml @@ -456,7 +456,7 @@ jobs: - windows-latest python-version: - "3.9" - - "3.11" + - "3.11.8" backend: - name: dask title: Dask @@ -490,7 +490,7 @@ jobs: extras: - postgres - geospatial - - python-version: "3.11" + - python-version: "3.11.8" backend: name: postgres title: PostgreSQL