chore: split e2e tests into multiple workflows (#808)

* split e2e.yaml into several workflows * chore: use custom actions to make workflows more DRY --------- Co-authored-by: Jon Perry <[email protected]>
defenseunicorns · Jul 30, 2024 · c993ad5 · c993ad5
1 parent 22c75cb
commit c993ad5
Show file tree

Hide file tree

Showing 11 changed files with 552 additions and 238 deletions.
diff --git a/.github/actions/lfai-core/action.yaml b/.github/actions/lfai-core/action.yaml
@@ -0,0 +1,28 @@
+name: setup-lfai-core
+description: "Setup Supabase and LFAI-API"
+
+runs:
+  using: composite
+  steps:
+    - name: Deploy Supabase
+      shell: bash
+      run: |
+        make build-supabase LOCAL_VERSION=e2e-test
+        docker image prune -af
+        uds zarf tools kubectl create namespace leapfrogai
+        uds zarf package deploy packages/supabase/zarf-package-supabase-amd64-e2e-test.tar.zst --confirm
+        rm packages/supabase/zarf-package-supabase-amd64-e2e-test.tar.zst
+
+    - name: Set environment variable
+      shell: bash
+      id: set-env-var
+      run: |
+        echo "ANON_KEY=$(uds zarf tools kubectl get secret supabase-bootstrap-jwt -n leapfrogai -o jsonpath='{.data.anon-key}' | base64 -d)" >> "$GITHUB_ENV"
+
+    - name: Deploy LFAI-API
+      shell: bash
+      run: |
+        make build-api LOCAL_VERSION=e2e-test
+        docker image prune -af
+        uds zarf package deploy packages/api/zarf-package-leapfrogai-api-amd64-e2e-test.tar.zst --confirm
+        rm packages/api/zarf-package-leapfrogai-api-amd64-e2e-test.tar.zst
diff --git a/.github/actions/python/action.yaml b/.github/actions/python/action.yaml
@@ -0,0 +1,14 @@
+name: setup-python
+description: "Setup Python and library dependencies"
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Python
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c #v5.0.0
+      with:
+        python-version-file: 'pyproject.toml'
+
+    - name: Install Python Deps
+      run: python -m pip install ".[dev]"
+      shell: bash
diff --git a/.github/actions/uds-cluster/action.yaml b/.github/actions/uds-cluster/action.yaml
@@ -0,0 +1,22 @@
+name: setup-uds
+description: "Setup UDS Cluster"
+
+inputs:
+  registry1Username:
+    description: Registry1 Username
+  registry1Password:
+    description: Registry1 Password
+
+runs:
+  using: composite
+  steps:
+    - name: Setup UDS Environment
+      uses: defenseunicorns/uds-common/.github/actions/setup@05f42bb3117b66ebef8c72ae050b34bce19385f5
+      with:
+        username: ${{ inputs.registry1Username }}
+        password: ${{ inputs.registry1Password }}
+
+    - name: Create UDS Cluster
+      shell: bash
+      run: |
+        uds deploy k3d-core-slim-dev:0.22.2 --confirm
diff --git a/.github/workflows/e2e-llama-cpp-python.yaml b/.github/workflows/e2e-llama-cpp-python.yaml
@@ -0,0 +1,85 @@
+# End-to-end testing that deploys Supabase and the API, and deploy/tests llama-cpp-python, text-embeddings, and whisper
+
+name: e2e-llama-cpp-python
+on:
+  pull_request:
+    types:
+      - ready_for_review
+      - review_requested
+      - synchronize
+      - milestoned
+    paths:
+      # Catch-all
+      - "**"
+
+      # Ignore updates to the .github directory, unless it's this current file
+      - "!.github/**"
+      - ".github/workflows/e2e-llama-cpp-python.yaml"
+
+      # Ignore docs and website things
+      - "!**.md"
+      - "!docs/**"
+      - "!adr/**"
+      - "!website/**"
+      - "!netlify.toml"
+
+      # Ignore updates to generic github metadata files
+      - "!CODEOWNERS"
+      - "!.gitignore"
+      - "!LICENSE"
+
+      # Ignore local development files
+      - "!.pre-commit-config.yaml"
+
+      # Ignore non e2e tests changes
+      - "!tests/pytest/**"
+
+      # Ignore LFAI-UI source code changes
+      - "!src/leapfrogai_ui/**"
+
+      # Ignore changes to unrelated packages
+      - "!packages/k3d-gpu/**"
+      - "!packages/repeater/**"
+      - "!packages/text-embeddings/**"
+      - "!packages/ui/**"
+      - "!packages/vllm/**"
+      - "!packages/whisper/**"
+
+concurrency:
+  group: e2e-llama-cpp-python-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e_llama:
+    runs-on: ai-ubuntu-big-boy-8-core
+    if: ${{ !github.event.pull_request.draft }}
+
+    steps:
+        - name: Checkout Repo
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - name: Setup Python
+          uses: ./.github/actions/python
+
+        - name: Setup UDS Cluster
+          uses: ./.github/actions/uds-cluster
+          with:
+            registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }}
+            registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }}
+
+        - name: Setup LFAI-API and Supabase
+          uses: ./.github/actions/lfai-core
+
+        ##########
+        # llama
+        ##########
+        - name: Deploy llama-cpp-python
+          run: |
+            make build-llama-cpp-python LOCAL_VERSION=e2e-test
+            docker image prune -af
+            uds zarf package deploy packages/llama-cpp-python/zarf-package-llama-cpp-python-amd64-e2e-test.tar.zst -l=trace --confirm
+            rm packages/llama-cpp-python/zarf-package-llama-cpp-python-amd64-e2e-test.tar.zst
+
+        - name: Test llama-cpp-python
+          run: |
+            python -m pytest ./tests/e2e/test_llama.py -v
diff --git a/.github/workflows/e2e-playwright.yaml b/.github/workflows/e2e-playwright.yaml
@@ -0,0 +1,120 @@
+# End-to-end testing that deploys and tests Supabase, API, UI, and VLLM
+
+name: e2e-playwright
+on:
+  pull_request:
+    types:
+      - ready_for_review
+      - review_requested
+      - synchronize
+      - milestoned
+    paths:
+      # Catch-all
+      - "**"
+
+      # Ignore updates to the .github directory, unless it's this current file
+      - "!.github/**"
+      - ".github/workflows/e2e-playwright.yaml"
+
+      # Ignore docs and website things
+      - "!**.md"
+      - "!docs/**"
+      - "!adr/**"
+      - "!website/**"
+      - "!netlify.toml"
+
+      # Ignore updates to generic github metadata files
+      - "!CODEOWNERS"
+      - "!.gitignore"
+      - "!LICENSE"
+
+      # Ignore local development files
+      - "!.pre-commit-config.yaml"
+
+      # Ignore non e2e tests changes
+      - "!tests/pytest/**"
+
+      # Ignore changes to unrelated packages
+      - "!packages/k3d-gpu/**"
+      - "!packages/llama-cpp-python/**"
+      - "!packages/repeater/**"
+      - "!packages/text-embeddings/**"
+      - "!packages/vllm/**"
+      - "!packages/whisper/**"
+
+
+
+concurrency:
+  group: e2e-playwright-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e_playwright:
+    runs-on: ai-ubuntu-big-boy-8-core
+    if: ${{ !github.event.pull_request.draft }}
+
+    steps:
+        - name: Checkout Repo
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - name: Setup Node
+          uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+          with:
+            node-version-file: 'src/leapfrogai_ui/package.json'
+
+        - name: Install UI/Playwright Dependencies
+          run: |
+            npm --prefix src/leapfrogai_ui ci
+            npx --prefix src/leapfrogai_ui playwright install
+
+        - name: Setup Python
+          uses: ./.github/actions/python
+
+        - name: Setup UDS Cluster
+          uses: ./.github/actions/uds-cluster
+          with:
+            registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }}
+            registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }}
+
+        - name: Setup LFAI-API and Supabase
+          uses: ./.github/actions/lfai-core
+
+        - name: Test Supabase
+          run: |
+            python -m pip install requests
+            python -m pytest ./tests/e2e/test_supabase.py -v
+
+        - name: Test API
+          run: |
+            python -m pip install requests
+            python -m pytest ./tests/e2e/test_api.py -v
+
+        ##########
+        # UI
+        ##########
+        - name: Deploy LFAI-UI
+          run: |
+            make build-ui LOCAL_VERSION=e2e-test
+            docker image prune -af
+            uds zarf package deploy packages/ui/zarf-package-leapfrogai-ui-amd64-e2e-test.tar.zst --confirm
+            rm packages/ui/zarf-package-leapfrogai-ui-amd64-e2e-test.tar.zst
+
+        # Run the playwright UI tests using the deployed Supabase endpoint and upload report as an artifact
+        - name: UI/API/Supabase E2E Playwright Tests
+          run: |
+            cp src/leapfrogai_ui/.env.example src/leapfrogai_ui/.env
+            TEST_ENV=CI PUBLIC_DISABLE_KEYCLOAK=true PUBLIC_SUPABASE_ANON_KEY=$ANON_KEY npm --prefix src/leapfrogai_ui run test:integration:ci
+
+        # Upload the Playwright report as an artifact
+        - name: Archive Playwright Report
+          uses: actions/upload-artifact@v4
+          if: ${{ !cancelled() }}
+          with:
+            name: playwright-report
+            path: src/leapfrogai_ui/e2e-report/
+            retention-days: 30
+
+        # The UI can be removed after the Playwright tests are finished
+        - name: Cleanup UI
+          run: |
+            uds zarf package remove leapfrogai-ui --confirm
diff --git a/.github/workflows/e2e-shim.yaml b/.github/workflows/e2e-shim.yaml
@@ -1,30 +1,7 @@
 # Catch all the things we ignore in the e2e workflow
 name: e2e Skip Shim
 on:
-  pull_request:
-    paths:
-      # Catch updates to the .github directory, unless it is the e2e.yaml files
-      - ".github/**"
-      - "!.github/workflows/e2e.yaml"
-
-      # Catch docs and website things
-      - "**.md"
-      - "adr/**"
-      - "docs/**"
-      - "website/**"
-      - "netlify.toml"
-
-      # Catch generic github metadata files
-      - "CODEOWNERS"
-      - ".gitignore"
-      - "LICENSE"
-      - ".pre-commit-config.yaml"
-
-      # Catch pytests
-      - "tests/pytest/**"
-
-      # Catch changes to the repeater model
-      - "packages/repeater/**"
+  pull_request
 
 
 permissions:
@@ -47,7 +24,31 @@ concurrency:
 
 
 jobs:
-  e2e:
+  e2e_llama:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skipped
+        run: |
+          echo skipped
+  e2e_playwright:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skipped
+        run: |
+          echo skipped
+  e2e_text_embeddings:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skipped
+        run: |
+          echo skipped
+  e2e_vllm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Skipped
+        run: |
+          echo skipped
+  e2e_whisper:
     runs-on: ubuntu-latest
     steps:
       - name: Skipped