From 59b9efff787f2e7a061881b2891464b4357c0fe6 Mon Sep 17 00:00:00 2001 From: Asim Biswal Date: Tue, 13 Jun 2023 00:20:24 +0000 Subject: [PATCH 1/4] update tests for cleanup/unnecessary installs --- .github/workflows/integration-test-local.yml | 5 +++++ .github/workflows/integration-test-multiple-sizes.yml | 5 +++++ .github/workflows/pytest.yml | 2 -- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test-local.yml b/.github/workflows/integration-test-local.yml index 8ae04a774..b35609351 100644 --- a/.github/workflows/integration-test-local.yml +++ b/.github/workflows/integration-test-local.yml @@ -90,6 +90,11 @@ jobs: skyplane config set usage_stats false - name: Deprovision run: skyplane deprovision + - name: Delete matching S3 buckets + run: | + for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do + aws s3api list-buckets --query "Buckets[?starts_with(Name, \`${pattern}\`) == \`true\`].Name" --output text | tr '\t' '\n' | while read bucket; do aws s3 rb "s3://$bucket" --force; done + done - name: Cleanup GCP service account if: always() run: gcloud iam service-accounts delete --quiet ${{ env.STRATEGY_UUID }}@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com diff --git a/.github/workflows/integration-test-multiple-sizes.yml b/.github/workflows/integration-test-multiple-sizes.yml index ca216daa8..87b6027d3 100644 --- a/.github/workflows/integration-test-multiple-sizes.yml +++ b/.github/workflows/integration-test-multiple-sizes.yml @@ -96,6 +96,11 @@ jobs: skyplane config set usage_stats false - name: Deprovision run: skyplane deprovision + - name: Delete matching S3 buckets + run: | + for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do + aws s3api list-buckets --query "Buckets[?starts_with(Name, \`${pattern}\`) == \`true\`].Name" --output text | tr '\t' '\n' | while read bucket; do aws s3 rb "s3://$bucket" --force; done + done - name: Cleanup GCP service account if: always() run: gcloud iam service-accounts delete --quiet ${{ env.STRATEGY_UUID }}@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 49e753645..de6e91980 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -87,8 +87,6 @@ jobs: run: | poetry install -E gateway -E solver -E aws -E azure -E gcp -E ibm poetry run pip install -r requirements-dev.txt - poetry run sudo apt install default-jdk - poetry run wget https://archive.apache.org/dist/hadoop/core/hadoop-3.3.0/hadoop-3.3.0.tar.gz -P /tmp && tar -xzf /tmp/hadoop-3.3.0.tar.gz -C /tmp && sudo mv /tmp/hadoop-3.3.0 /usr/local/hadoop && rm /tmp/hadoop-3.3.0.tar.gz if: steps.cache.outputs.cache-hit != 'true' - name: Run cloud tests env: From c4edbd3350739f3fd426c74cc24b12c123f7f018 Mon Sep 17 00:00:00 2001 From: Asim Biswal Date: Tue, 13 Jun 2023 23:53:48 +0000 Subject: [PATCH 2/4] removed hadoop jdk from test-gcp as well --- .github/workflows/pytest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index de6e91980..b7ae05842 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -154,7 +154,6 @@ jobs: poetry install -E gateway -E solver -E aws -E azure -E gcp -E ibm poetry run pip install -r requirements-dev.txt poetry run sudo apt install default-jdk - poetry run wget https://archive.apache.org/dist/hadoop/core/hadoop-3.3.0/hadoop-3.3.0.tar.gz -P /tmp && tar -xzf /tmp/hadoop-3.3.0.tar.gz -C /tmp && sudo mv /tmp/hadoop-3.3.0 /usr/local/hadoop && rm /tmp/hadoop-3.3.0.tar.gz if: steps.cache.outputs.cache-hit != 'true' - id: 'auth' uses: 'google-github-actions/auth@v0' From 8e130aefe9469274a1d3cbec896ad2778cd66b5e Mon Sep 17 00:00:00 2001 From: Asim Biswal Date: Wed, 14 Jun 2023 00:38:28 +0000 Subject: [PATCH 3/4] added poetry run to failing tests --- .github/workflows/integration-test-local.yml | 8 ++++---- .github/workflows/integration-test-multiple-sizes.yml | 8 ++++---- .github/workflows/integration-test.yml | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/integration-test-local.yml b/.github/workflows/integration-test-local.yml index b35609351..a3e740d70 100644 --- a/.github/workflows/integration-test-local.yml +++ b/.github/workflows/integration-test-local.yml @@ -85,11 +85,11 @@ jobs: credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}' - name: Skyplane init run: | - skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }} - skyplane init -y --disable-config-azure --disable-config-cloudflare - skyplane config set usage_stats false + poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }} + poetry run skyplane init -y --disable-config-azure --disable-config-cloudflare + poetry run skyplane config set usage_stats false - name: Deprovision - run: skyplane deprovision + run: poetry run skyplane deprovision - name: Delete matching S3 buckets run: | for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do diff --git a/.github/workflows/integration-test-multiple-sizes.yml b/.github/workflows/integration-test-multiple-sizes.yml index 87b6027d3..b2e308732 100644 --- a/.github/workflows/integration-test-multiple-sizes.yml +++ b/.github/workflows/integration-test-multiple-sizes.yml @@ -91,11 +91,11 @@ jobs: credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}' - name: Skyplane init run: | - skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }} - skyplane init -y --disable-config-azure --disable-config-cloudflare - skyplane config set usage_stats false + poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }} + poetry run skyplane init -y --disable-config-azure --disable-config-cloudflare + poetry run skyplane config set usage_stats false - name: Deprovision - run: skyplane deprovision + run: poetry run skyplane deprovision - name: Delete matching S3 buckets run: | for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1bfcd7ab6..eb7b4822c 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -90,11 +90,11 @@ jobs: credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}' - name: Skyplane init run: | - skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }} - skyplane init -y --disable-config-azure - skyplane config set usage_stats false + poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }} + poetry run skyplane init -y --disable-config-azure + poetry run skyplane config set usage_stats false - name: Deprovision - run: skyplane deprovision + run: poetry run skyplane deprovision - name: Delete matching S3 buckets run: | for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do From 68f9550c5de0f5df10cf40560dbff6750daca88d Mon Sep 17 00:00:00 2001 From: Asim Biswal Date: Wed, 14 Jun 2023 00:53:58 +0000 Subject: [PATCH 4/4] poetry install on deprovision step --- .github/workflows/integration-test-local.yml | 12 ++++++++++-- .../workflows/integration-test-multiple-sizes.yml | 8 ++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-test-local.yml b/.github/workflows/integration-test-local.yml index a3e740d70..b10a66775 100644 --- a/.github/workflows/integration-test-local.yml +++ b/.github/workflows/integration-test-local.yml @@ -73,12 +73,20 @@ jobs: STRATEGY_UUID: itest-d-${{ github.run_id }}-${{ github.run_attempt }} steps: - uses: actions/checkout@v1 + - name: Install poetry + run: pipx install poetry - name: Set up Python 3.10 uses: actions/setup-python@v4 with: python-version: "3.10" - - name: Install skyplane from pypi - run: pip install skyplane[aws,azure,gcp] + cache: "poetry" + - name: Set Poetry config + run: | + poetry config virtualenvs.in-project false + poetry config virtualenvs.path ~/.virtualenvs + - name: Install dependencies + run: poetry install -E aws -E azure -E gcp + if: steps.cache.outputs.cache-hit != 'true' - id: 'auth' uses: 'google-github-actions/auth@v1' with: diff --git a/.github/workflows/integration-test-multiple-sizes.yml b/.github/workflows/integration-test-multiple-sizes.yml index b2e308732..e3c4b1c4a 100644 --- a/.github/workflows/integration-test-multiple-sizes.yml +++ b/.github/workflows/integration-test-multiple-sizes.yml @@ -79,12 +79,16 @@ jobs: STRATEGY_UUID: itest-d-${{ github.run_id }}-${{ github.run_attempt }} steps: - uses: actions/checkout@v1 + - name: Install poetry + run: pipx install poetry - name: Set up Python 3.10 uses: actions/setup-python@v4 with: python-version: "3.10" - - name: Install skyplane from pypi - run: pip install skyplane[aws,azure,gcp] + cache: "poetry" + - name: Install dependencies + run: poetry install -E aws -E azure -E gcp + if: steps.cache.outputs.cache-hit != 'true' - id: 'auth' uses: 'google-github-actions/auth@v1' with: