Faster Tests (DjangoCon) (#1602)

* Running tests the sentry-ruby way (splitting up into multiple yaml files. Created a script to split tox.ini into multiple yaml files automatically) * Cleaning up the yaml file in general. * Removed PyPy from the test suite because it was never run. We have to reevaluate support for PyPy. This fixes #1499
getsentry · Sep 19, 2022 · f5ee56b · f5ee56b
1 parent b36d84a
commit f5ee56b
Show file tree

Hide file tree

Showing 33 changed files with 1,806 additions and 96 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -32,81 +32,19 @@ jobs:
           pip install tox
           tox -e linters
 
-  test:
-    name: Run Tests
-    runs-on: ${{ matrix.linux-version }}
-    timeout-minutes: 45
-    continue-on-error: true
-    strategy:
-      matrix:
-        linux-version: [ubuntu-latest]
-        python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"]
-        include:
-          # GHA doesn't host the combo of python 3.4 and ubuntu-latest (which is
-          # currently 20.04), so run just that one under 18.04. (See
-          # https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
-          # for a listing of supported python/os combos.)
-          - linux-version: ubuntu-18.04
-            python-version: "3.4"
-
-    services:
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Set health checks to wait until redis has started
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps port 6379 on service container to the host
-          - 6379:6379
-
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: sentry
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        # Maps tcp port 5432 on service container to the host
-        ports:
-          - 5432:5432
-
-    env:
-      SENTRY_PYTHON_TEST_POSTGRES_USER: postgres
-      SENTRY_PYTHON_TEST_POSTGRES_PASSWORD: sentry
-      SENTRY_PYTHON_TEST_POSTGRES_NAME: ci_test
+  check-ci-config:
+    name: Check CI config
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
 
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
-        run: |
-          pip install codecov tox
+          python-version: 3.9
 
-      - name: Run Tests
-        env:
-          CI_PYTHON_VERSION: ${{ matrix.python-version }}
-        timeout-minutes: 45
-        run: |
-          coverage erase
-          ./scripts/runtox.sh '' --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
-          codecov --file coverage.xml
+      - run: |
+          python scripts/split-tox-gh-actions/split-tox-gh-actions.py --fail-on-changes
 
   build_lambda_layer:
     name: Build Package

diff --git a/.github/workflows/test-common.yml b/.github/workflows/test-common.yml
@@ -0,0 +1,72 @@
+name: Test Common
+
+on:
+  push:
+    branches:
+      - master
+      - release/**
+
+  pull_request:
+
+permissions:
+  contents: read
+
+env:
+  BUILD_CACHE_KEY: ${{ github.sha }}
+  CACHED_BUILD_PATHS: |
+    ${{ github.workspace }}/dist-serverless
+
+jobs:
+  test:
+    name: Test Python ${{ matrix.python-version }}, ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    continue-on-error: true
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: sentry
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        # Maps tcp port 5432 on service container to the host
+        ports:
+          - 5432:5432
+    env:
+      SENTRY_PYTHON_TEST_POSTGRES_USER: postgres
+      SENTRY_PYTHON_TEST_POSTGRES_PASSWORD: sentry
+      SENTRY_PYTHON_TEST_POSTGRES_NAME: ci_test
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Test Env
+        env:
+          PGHOST: localhost
+          PGPASSWORD: sentry
+        run: |
+          pip install codecov tox
+
+      - name: Run Tests
+        env:
+          CI_PYTHON_VERSION: ${{ matrix.python-version }}
+        timeout-minutes: 45
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          coverage erase
+
+          ./scripts/runtox.sh "py${{ matrix.python-version }}$" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch --ignore=tests/integrations
+          coverage combine .coverage*
+          coverage xml -i
+          codecov --file coverage.xml
diff --git a/.github/workflows/test-integration-aiohttp.yml b/.github/workflows/test-integration-aiohttp.yml
@@ -0,0 +1,56 @@
+name: Test aiohttp
+
+on:
+  push:
+    branches:
+      - master
+      - release/**
+
+  pull_request:
+
+permissions:
+  contents: read
+
+env:
+  BUILD_CACHE_KEY: ${{ github.sha }}
+  CACHED_BUILD_PATHS: |
+    ${{ github.workspace }}/dist-serverless
+
+jobs:
+  test:
+    name: aiohttp, python ${{ matrix.python-version }}, ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    continue-on-error: true
+
+    strategy:
+      matrix:
+        python-version: ["3.7","3.8","3.9","3.10"]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Test Env
+        env:
+          PGHOST: localhost
+          PGPASSWORD: sentry
+        run: |
+          pip install codecov tox
+
+      - name: Test aiohttp
+        env:
+          CI_PYTHON_VERSION: ${{ matrix.python-version }}
+        timeout-minutes: 45
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          coverage erase
+
+          ./scripts/runtox.sh "${{ matrix.python-version }}-aiohttp" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
+          coverage combine .coverage*
+          coverage xml -i
+          codecov --file coverage.xml
diff --git a/.github/workflows/test-integration-asgi.yml b/.github/workflows/test-integration-asgi.yml
@@ -0,0 +1,56 @@
+name: Test asgi
+
+on:
+  push:
+    branches:
+      - master
+      - release/**
+
+  pull_request:
+
+permissions:
+  contents: read
+
+env:
+  BUILD_CACHE_KEY: ${{ github.sha }}
+  CACHED_BUILD_PATHS: |
+    ${{ github.workspace }}/dist-serverless
+
+jobs:
+  test:
+    name: asgi, python ${{ matrix.python-version }}, ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    continue-on-error: true
+
+    strategy:
+      matrix:
+        python-version: ["3.7","3.8","3.9","3.10"]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Test Env
+        env:
+          PGHOST: localhost
+          PGPASSWORD: sentry
+        run: |
+          pip install codecov tox
+
+      - name: Test asgi
+        env:
+          CI_PYTHON_VERSION: ${{ matrix.python-version }}
+        timeout-minutes: 45
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          coverage erase
+
+          ./scripts/runtox.sh "${{ matrix.python-version }}-asgi" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
+          coverage combine .coverage*
+          coverage xml -i
+          codecov --file coverage.xml
diff --git a/.github/workflows/test-integration-aws_lambda.yml b/.github/workflows/test-integration-aws_lambda.yml
@@ -0,0 +1,56 @@
+name: Test aws_lambda
+
+on:
+  push:
+    branches:
+      - master
+      - release/**
+
+  pull_request:
+
+permissions:
+  contents: read
+
+env:
+  BUILD_CACHE_KEY: ${{ github.sha }}
+  CACHED_BUILD_PATHS: |
+    ${{ github.workspace }}/dist-serverless
+
+jobs:
+  test:
+    name: aws_lambda, python ${{ matrix.python-version }}, ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    continue-on-error: true
+
+    strategy:
+      matrix:
+        python-version: ["3.7"]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Test Env
+        env:
+          PGHOST: localhost
+          PGPASSWORD: sentry
+        run: |
+          pip install codecov tox
+
+      - name: Test aws_lambda
+        env:
+          CI_PYTHON_VERSION: ${{ matrix.python-version }}
+        timeout-minutes: 45
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          coverage erase
+
+          ./scripts/runtox.sh "${{ matrix.python-version }}-aws_lambda" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
+          coverage combine .coverage*
+          coverage xml -i
+          codecov --file coverage.xml
diff --git a/.github/workflows/test-integration-beam.yml b/.github/workflows/test-integration-beam.yml
@@ -0,0 +1,56 @@
+name: Test beam
+
+on:
+  push:
+    branches:
+      - master
+      - release/**
+
+  pull_request:
+
+permissions:
+  contents: read
+
+env:
+  BUILD_CACHE_KEY: ${{ github.sha }}
+  CACHED_BUILD_PATHS: |
+    ${{ github.workspace }}/dist-serverless
+
+jobs:
+  test:
+    name: beam, python ${{ matrix.python-version }}, ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    continue-on-error: true
+
+    strategy:
+      matrix:
+        python-version: ["3.7"]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Test Env
+        env:
+          PGHOST: localhost
+          PGPASSWORD: sentry
+        run: |
+          pip install codecov tox
+
+      - name: Test beam
+        env:
+          CI_PYTHON_VERSION: ${{ matrix.python-version }}
+        timeout-minutes: 45
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          coverage erase
+
+          ./scripts/runtox.sh "${{ matrix.python-version }}-beam" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
+          coverage combine .coverage*
+          coverage xml -i
+          codecov --file coverage.xml