From f33d0009819cf5927c0b0a0be7575e6fcce400ca Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 8 Jan 2025 19:19:56 +0100 Subject: [PATCH] install ibis outside of poetry --- .github/workflows/test_dbt_cloud.yml | 1 + .github/workflows/test_destination_athena.yml | 3 ++- .github/workflows/test_destination_athena_iceberg.yml | 3 ++- .github/workflows/test_destination_bigquery.yml | 3 ++- .github/workflows/test_destination_clickhouse.yml | 3 ++- .github/workflows/test_destination_databricks.yml | 3 ++- .github/workflows/test_destination_dremio.yml | 3 ++- .github/workflows/test_destination_lancedb.yml | 1 + .github/workflows/test_destination_motherduck.yml | 3 ++- .github/workflows/test_destination_mssql.yml | 3 ++- .github/workflows/test_destination_qdrant.yml | 1 + .github/workflows/test_destination_snowflake.yml | 3 ++- ...chemy_destinations.yml => test_destination_sqlalchemy.yml} | 3 ++- .github/workflows/test_destination_synapse.yml | 3 ++- .github/workflows/test_destinations.yml | 3 ++- .github/workflows/test_local_destinations.yml | 4 +--- .github/workflows/test_local_sources.yml | 1 + 17 files changed, 29 insertions(+), 15 deletions(-) rename .github/workflows/{test_sqlalchemy_destinations.yml => test_destination_sqlalchemy.yml} (91%) diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml index 5b57dc77c3..d88c1b914d 100644 --- a/.github/workflows/test_dbt_cloud.yml +++ b/.github/workflows/test_dbt_cloud.yml @@ -52,6 +52,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 03eb7f9434..d1a0510088 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -56,6 +56,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -67,7 +68,7 @@ jobs: - name: Install dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 3412e789e3..bfa9f0a1b7 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -56,6 +56,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -67,7 +68,7 @@ jobs: - name: Install dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index eb8b63f757..04994661fc 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -55,6 +55,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -66,7 +67,7 @@ jobs: - name: Install dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destination_clickhouse.yml b/.github/workflows/test_destination_clickhouse.yml index 46464ea462..cde0c85802 100644 --- a/.github/workflows/test_destination_clickhouse.yml +++ b/.github/workflows/test_destination_clickhouse.yml @@ -52,6 +52,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -61,7 +62,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp - name: Install dependencies - run: poetry install --no-interaction -E clickhouse --with providers -E parquet --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E clickhouse --with providers -E parquet --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index c1609de863..0c5dbecbb5 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -55,6 +55,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -64,7 +65,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp - name: Install dependencies - run: poetry install --no-interaction -E databricks -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E databricks -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destination_dremio.yml b/.github/workflows/test_destination_dremio.yml index 4bc48c54db..5a0795eeb6 100644 --- a/.github/workflows/test_destination_dremio.yml +++ b/.github/workflows/test_destination_dremio.yml @@ -56,6 +56,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -65,7 +66,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp - name: Install dependencies - run: poetry install --no-interaction -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - run: | poetry run pytest tests/load --ignore tests/load/sources diff --git a/.github/workflows/test_destination_lancedb.yml b/.github/workflows/test_destination_lancedb.yml index 6be89d3de3..960d0cacf9 100644 --- a/.github/workflows/test_destination_lancedb.yml +++ b/.github/workflows/test_destination_lancedb.yml @@ -53,6 +53,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies diff --git a/.github/workflows/test_destination_motherduck.yml b/.github/workflows/test_destination_motherduck.yml index db81131266..bfc11ee545 100644 --- a/.github/workflows/test_destination_motherduck.yml +++ b/.github/workflows/test_destination_motherduck.yml @@ -55,6 +55,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -64,7 +65,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-motherduck - name: Install dependencies - run: poetry install --no-interaction -E motherduck -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E motherduck -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 6fdd7a5bc5..aaa07358a6 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -60,6 +60,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -69,7 +70,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp - name: Install dependencies - run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index c35a171bce..01f9cc44d6 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -54,6 +54,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 73a2a8f6e7..666a47cb70 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -55,6 +55,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -64,7 +65,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp - name: Install dependencies - run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_sqlalchemy_destinations.yml b/.github/workflows/test_destination_sqlalchemy.yml similarity index 91% rename from .github/workflows/test_sqlalchemy_destinations.yml rename to .github/workflows/test_destination_sqlalchemy.yml index 1f00373674..36b9c47182 100644 --- a/.github/workflows/test_sqlalchemy_destinations.yml +++ b/.github/workflows/test_destination_sqlalchemy.yml @@ -77,6 +77,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -86,7 +87,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations - name: Install dependencies - run: poetry install --no-interaction -E parquet -E filesystem -E sqlalchemy -E cli --with sentry-sdk --with pipeline,ibis && poetry run pip install mysqlclient && poetry run pip install "sqlalchemy==${{ matrix.sqlalchemy }}" + run: poetry install --no-interaction -E parquet -E filesystem -E sqlalchemy -E cli --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] && poetry run pip install mysqlclient && poetry run pip install "sqlalchemy==${{ matrix.sqlalchemy }}" - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index 8f6bf1eb29..14d1931fc8 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -58,6 +58,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies @@ -67,7 +68,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp - name: Install dependencies - run: poetry install --no-interaction -E synapse -E parquet --with sentry-sdk --with pipeline,ibis + run: poetry install --no-interaction -E synapse -E parquet --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index a9306c2f9c..3894130c39 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -67,6 +67,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 # - name: Load cached venv # id: cached-poetry-dependencies @@ -77,7 +78,7 @@ jobs: # key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-redshift - name: Install dependencies - run: poetry install --no-interaction -E redshift -E postgis -E postgres -E gs -E s3 -E az -E parquet -E duckdb -E cli -E filesystem --with sentry-sdk --with pipeline,ibis -E deltalake -E pyiceberg + run: poetry install --no-interaction -E redshift -E postgis -E postgres -E gs -E s3 -E az -E parquet -E duckdb -E cli -E filesystem --with sentry-sdk --with pipeline -E deltalake -E pyiceberg && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: Upgrade sqlalchemy run: poetry run pip install sqlalchemy==2.0.18 # minimum version required by `pyiceberg` diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 776a85a20d..3a1d13975d 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -96,9 +96,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations - name: Install dependencies - run: | - poetry install --no-interaction -E postgres -E postgis -E duckdb -E parquet -E filesystem -E cli -E weaviate -E qdrant -E sftp --with sentry-sdk --with pipeline -E deltalake -E pyiceberg - poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] + run: poetry install --no-interaction -E postgres -E postgis -E duckdb -E parquet -E filesystem -E cli -E weaviate -E qdrant -E sftp --with sentry-sdk --with pipeline -E deltalake -E pyiceberg && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] - name: Upgrade sqlalchemy run: poetry run pip install sqlalchemy==2.0.18 # minimum version required by `pyiceberg` diff --git a/.github/workflows/test_local_sources.yml b/.github/workflows/test_local_sources.yml index 39689f5c85..95a7023dbf 100644 --- a/.github/workflows/test_local_sources.yml +++ b/.github/workflows/test_local_sources.yml @@ -73,6 +73,7 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + version: 1.8.5 - name: Load cached venv id: cached-poetry-dependencies