feat(integration): Add initial "integration tests"

.github/workflows/integration-tests.yml

@@ -0,0 +1,114 @@
+name: Integration tests
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "backend/**"
+      - "keycloak/**"
+      - "kubernetes/**"
+      - "website/**"
+      - "deploy.py"
+      - "integration-tests/**"
+      - ".github/scripts/**"
+      - ".github/workflows/**"
+  push:
+    branches:
+      - main
+jobs:
+  integration-tests:
+    permissions:
+      packages: read
+      contents: read
+      checks: read
+      actions: read # Required by workflow-telemetry-action
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      sha: ${{ github.event.pull_request.head.sha || github.sha }}
+    steps:
+      - name: Shorten sha
+        run: echo "sha=${sha::7}" >> $GITHUB_ENV
+      - name: Collect Workflow Telemetry
+        uses: catchpoint/workflow-telemetry-action@v2
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install k3d
+        run: |
+          curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+          k3d version
+      - uses: azure/setup-helm@v4
+      - name: Create k3d cluster
+        run: |
+          ./deploy.py --verbose cluster
+      - name: Deploy with helm
+        run: |
+          ./deploy.py --verbose helm --branch ${{ github.ref_name }} --sha ${{ env.sha }} --for-e2e
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: ./website/.nvmrc
+      - name: Cache .npm
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('integration-tests/**/package-lock.json') }}
+      - name: Install dependencies
+        run: cd integration-tests && npm i
+      - name: Get Installed Playwright Version
+        id: playwright-version
+        run: cd integration-tests && echo "PLAYWRIGHT_VERSION=$(node -e "console.log(require('./package-lock.json').packages['node_modules/@playwright/test'].version)")" >> $GITHUB_ENV
+      - name: Cache Playwright Browsers
+        uses: actions/cache@v4
+        id: playwright-cache
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
+      - name: Install Playwright Browsers and System Dependencies
+        run: cd integration-tests && npx playwright install --with-deps
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+      - name: Install only System Dependencies
+        run: cd website && npx playwright install-deps
+        if: steps.playwright-cache.outputs.cache-hit == 'true'
+      - name: Wait for the pods to be ready
+        run: ./.github/scripts/wait_for_pods_to_be_ready.py --timeout 120
+      - name: Sleep for 10 secs
+        run: sleep 10
+      - name: Run Integration test
+        run: |
+          set -o pipefail
+          cd integration-tests && npx playwright test 2>&1 | tee output.txt
+          EXIT_CODE=$?
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          sed -n '/Running [0-9]\+ tests/,$p' output.txt >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          exit $EXIT_CODE
+      - uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: playwright-report
+          path: integration-tests/playwright-report/
+          retention-days: 30
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results
+          path: integration-tests/test-results/
+          retention-days: 30
+      - name: List running pods
+        if: ${{ !cancelled() }}
+        run: kubectl get pods --all-namespaces
+      - name: Describe pods
+        if: ${{ !cancelled() }}
+        run: kubectl describe pods -l app=loculus
+      - name: Show events
+        if: ${{ !cancelled() }}
+        run: kubectl get events
+      - name: Save logs from all containers to file
+        if: ${{ !cancelled() }}
+        run: ./.github/scripts/collect_kubernetes_logs.sh
+      - name: Upload Kubernetes logs
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: kubernetes-logs
+          path: kubernetes_logs/

deploy.py

@@ -217,6 +217,7 @@ def handle_helm():
 
     if args.for_e2e or args.dev:
         parameters += ["-f", HELM_CHART_DIR / "values_e2e_and_dev.yaml"]
+        parameters += ["--set", "auth.verifyEmail=false"]
     if args.sha:
         parameters += ["--set", f"sha={args.sha[:7]}"]
 

integration-tests/.gitignore

@@ -0,0 +1,5 @@
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

integration-tests/README.md

@@ -0,0 +1,9 @@
+# Integration tests
+
+These are tests of the full Loculus system, following sequences through submission to preprocessing to release.
+
+## Principles
+
+Here are some current guiding principles for these tests:
+- Only use facilities users could use (primarily browser interaction), rather than setting things up with backend calls. This makes it easy for others to understand the tests because they can follow them in the browser.
+- All tests should be able to run in parallel. Mostly this can be carried out by creating a separate user/group for each test.

integration-tests/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "integration_tests",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {},
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "devDependencies": {
+    "@playwright/test": "^1.48.2",
+    "@types/node": "^22.8.3",
+    "@types/uuid": "^10.0.0",
+    "uuid": "^11.0.2"
+  }
+}

integration-tests/playwright.config.ts

@@ -0,0 +1,80 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+/*
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+    */
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://127.0.0.1:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});

integration-tests/run.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+ ../deploy.py cluster --delete
+ ../deploy.py cluster
+ ../deploy.py helm --for-e2e
+ python ../.github/scripts/wait_for_pods_to_be_ready.py

integration-tests/tests/config/test.config.ts

@@ -0,0 +1,13 @@
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  testDir: '../specs',
+  use: {
+    baseURL: 'http://localhost:3000',
+    trace: 'on-first-retry',
+    screenshot: 'only-on-failure',
+  },
+  reporter: [['html'], ['list']],
+};
+
+export default config;