Update tests for bucket cleanup + remove unnecessary installs (#865)

S3 currently does not have proper bucket cleanup after testing leading to `TooManyBuckets` errors on tests after a certain point. This adds the cleanup logic to every integration test. Furthermore, the hadoop jdk installs were removed as they should not be required anymore.
skyplane-project · Jun 22, 2023 · cc6493c · cc6493c
1 parent e181521
commit cc6493c
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 19 deletions.
diff --git a/.github/workflows/integration-test-local.yml b/.github/workflows/integration-test-local.yml
@@ -73,23 +73,36 @@ jobs:
       STRATEGY_UUID: itest-d-${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - uses: actions/checkout@v1
+      - name: Install poetry
+        run: pipx install poetry
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
-      - name: Install skyplane from pypi
-        run: pip install skyplane[aws,azure,gcp]
+          cache: "poetry"
+      - name: Set Poetry config
+        run: |
+          poetry config virtualenvs.in-project false
+          poetry config virtualenvs.path ~/.virtualenvs
+      - name: Install dependencies
+        run: poetry install -E aws -E azure -E gcp
+        if: steps.cache.outputs.cache-hit != 'true'
       - id: 'auth'
         uses: 'google-github-actions/auth@v1'
         with:
           credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}'
       - name: Skyplane init
         run: |
-          skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
-          skyplane init -y --disable-config-azure
-          skyplane config set usage_stats false
+          poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
+          poetry run skyplane init -y --disable-config-azure --disable-config-cloudflare
+          poetry run skyplane config set usage_stats false
       - name: Deprovision
-        run: skyplane deprovision
+        run: poetry run skyplane deprovision
+      - name: Delete matching S3 buckets
+        run: |
+          for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do
+            aws s3api list-buckets --query "Buckets[?starts_with(Name, \`${pattern}\`) == \`true\`].Name" --output text | tr '\t' '\n' | while read bucket; do aws s3 rb "s3://$bucket" --force; done
+          done
       - name: Cleanup GCP service account
         if: always()
         run: gcloud iam service-accounts delete --quiet ${{ env.STRATEGY_UUID }}@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
diff --git a/.github/workflows/integration-test-multiple-sizes.yml b/.github/workflows/integration-test-multiple-sizes.yml
@@ -79,23 +79,32 @@ jobs:
       STRATEGY_UUID: itest-d-${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - uses: actions/checkout@v1
+      - name: Install poetry
+        run: pipx install poetry
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
-      - name: Install skyplane from pypi
-        run: pip install skyplane[aws,azure,gcp]
+          cache: "poetry"
+      - name: Install dependencies
+        run: poetry install -E aws -E azure -E gcp
+        if: steps.cache.outputs.cache-hit != 'true'
       - id: 'auth'
         uses: 'google-github-actions/auth@v1'
         with:
           credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}'
       - name: Skyplane init
         run: |
-          skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
-          skyplane init -y --disable-config-azure
-          skyplane config set usage_stats false
+          poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
+          poetry run skyplane init -y --disable-config-azure --disable-config-cloudflare
+          poetry run skyplane config set usage_stats false
       - name: Deprovision
-        run: skyplane deprovision
+        run: poetry run skyplane deprovision
+      - name: Delete matching S3 buckets
+        run: |
+          for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do
+            aws s3api list-buckets --query "Buckets[?starts_with(Name, \`${pattern}\`) == \`true\`].Name" --output text | tr '\t' '\n' | while read bucket; do aws s3 rb "s3://$bucket" --force; done
+          done
       - name: Cleanup GCP service account
         if: always()
         run: gcloud iam service-accounts delete --quiet ${{ env.STRATEGY_UUID }}@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -90,11 +90,11 @@ jobs:
           credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}'
       - name: Skyplane init
         run: |
-          skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
-          skyplane init -y --disable-config-azure
-          skyplane config set usage_stats false
+          poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
+          poetry run skyplane init -y --disable-config-azure
+          poetry run skyplane config set usage_stats false
       - name: Deprovision
-        run: skyplane deprovision
+        run: poetry run skyplane deprovision
       - name: Delete matching S3 buckets
         run: |
           for pattern in "test-skyplane-" "skyplane-integration-us-east-1-" "integrationus-east-1-"; do

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -87,8 +87,6 @@ jobs:
         run: |
           poetry install -E gateway -E solver -E aws -E azure -E gcp -E ibm
           poetry run pip install -r requirements-dev.txt
-          poetry run sudo apt install default-jdk
-          poetry run wget https://archive.apache.org/dist/hadoop/core/hadoop-3.3.0/hadoop-3.3.0.tar.gz -P /tmp && tar -xzf /tmp/hadoop-3.3.0.tar.gz -C /tmp && sudo mv /tmp/hadoop-3.3.0 /usr/local/hadoop && rm /tmp/hadoop-3.3.0.tar.gz
         if: steps.cache.outputs.cache-hit != 'true'
       - name: Run cloud tests
         env:
@@ -156,7 +154,6 @@ jobs:
           poetry install -E gateway -E solver -E aws -E azure -E gcp -E ibm
           poetry run pip install -r requirements-dev.txt
           poetry run sudo apt install default-jdk
-          poetry run wget https://archive.apache.org/dist/hadoop/core/hadoop-3.3.0/hadoop-3.3.0.tar.gz -P /tmp && tar -xzf /tmp/hadoop-3.3.0.tar.gz -C /tmp && sudo mv /tmp/hadoop-3.3.0 /usr/local/hadoop && rm /tmp/hadoop-3.3.0.tar.gz
         if: steps.cache.outputs.cache-hit != 'true'
       - id: 'auth'
         uses: 'google-github-actions/auth@v0'