Cache charmcraft pack container, skip unstable tests except on schedule

.github/workflows/ci.yaml

@@ -1,7 +1,13 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
 name: Tests
+
 on:
-  workflow_call:
   pull_request:
+  schedule:
+    - cron: '53 0 * * *' # Daily at 00:53 UTC
+  # Triggered on push to branch "main" by .github/workflows/release.yaml
+  workflow_call:
 
 jobs:
   lint:
@@ -10,21 +16,23 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
-        run: python -m pip install tox
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
+        run: python3 -m pip install tox
       - name: Run linters
-        run: tox -e lint
+        run: tox run -e lint
 
   unit-test:
     name: Unit tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
-        run: python -m pip install tox
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
+        run: python3 -m pip install tox
       - name: Run tests
-        run: tox -e unit
+        run: tox run -e unit
 
   lib-check:
     name: Check libraries
@@ -40,150 +48,57 @@ jobs:
           credentials: "${{ secrets.CHARMHUB_TOKEN }}"
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 
-  integration-standalone:
-    name: Integration tests for standalone charm
-    needs:
-      - lib-check
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-      # This part sets the model name to "testing" (used in test run and logdump)
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Run pgbouncer standalone integration tests
-        run: tox -e standalone-integration
-      - name: Dump logs
-        uses: canonical/charm-logdump-action@main
-        if: failure()
-        with:
-          app: pgbouncer-k8s
-
-  integration-backend:
-    name: Integration tests for backend relation
-    needs:
-      - lib-check
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-      # This part sets the model name to "testing" (used in test run and logdump)
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Run backend integration tests
-        run: tox -e backend-integration
-      - name: Dump logs
-        uses: canonical/charm-logdump-action@main
-        if: failure()
-        with:
-          app: pgbouncer-k8s
-
-  integration-legacy-client-relations:
-    name: Integration tests for legacy client relations
-    needs:
-      - lib-check
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-      # This part sets the model name to "testing" (used in test run and logdump)
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Run legacy integration tests
-        run: tox -e legacy-client-relation-integration
-      - name: Dump logs
-        uses: canonical/charm-logdump-action@main
-        if: failure()
-        with:
-          app: pgbouncer-k8s
-
-  integration-client-relations:
-    name: Integration tests for updated client relations
-    needs:
-      - lib-check
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-      # This part sets the model name to "testing" (used in test run and logdump)
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Run client integration tests
-        run: tox -e client-relation-integration
-      - name: Dump logs
-        uses: canonical/charm-logdump-action@main
-        if: failure()
-        with:
-          app: pgbouncer-k8s
+  build:
+    name: Build charms
+    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1
 
-  integration-scaling:
-    name: Integration tests for scaling pgbouncer
+  integration-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        tox-environments:
+          - standalone-integration
+          - backend-integration
+          - legacy-client-relation-integration
+          - client-relation-integration
+          - scaling-integration
+          - tls-integration
+    name: ${{ matrix.tox-environments }}
     needs:
       - lib-check
       - lint
       - unit-test
+      - build
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup operator environment
-      # This part sets the model name to "testing" (used in test run and logdump)
+        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
+          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed
           bootstrap-options: "--agent-version 2.9.29"
-      - name: Run scaling integration tests
-        run: tox -e scaling-integration
-      - name: Dump logs
-        uses: canonical/charm-logdump-action@main
-        if: failure()
-        with:
-          app: pgbouncer-k8s
-
-  integration-test-microk8s-tls:
-    name: Integration tests for TLS
-    needs:
-      - lib-check
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-      # This part sets the model name to "testing" (used in test run and logdump)
-        uses: charmed-kubernetes/actions-operator@main
+      - name: Download packed charm(s)
+        uses: actions/download-artifact@v3
         with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed.
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Run TLS integration tests
-        run: tox -e tls-integration
+          name: ${{ needs.build.outputs.artifact-name }}
+      - name: Select tests
+        id: select-tests
+        run: |
+          if [ "${{ github.event_name }}" == "schedule" ]
+          then
+            echo Running unstable and stable tests
+            echo "mark_expression=" >> $GITHUB_OUTPUT
+          else
+            echo Skipping unstable tests
+            echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
+          fi
+      - name: Run integration tests
+        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}'
+        env:
+          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
       - name: Dump logs
         uses: canonical/charm-logdump-action@main
         if: failure()

CONTRIBUTING.md

@@ -31,8 +31,8 @@ juju add-model dev
 juju model-config logging-config="<root>=INFO;unit=DEBUG"
 
 # initialise an environment using tox
-tox --notest -e unit
-source .tox/unit/bin/activate
+tox devenv -e integration
+source venv/bin/activate
 
 # export kubernetes config to lightkube, for integration testing.
 microk8s config > ~/.kube/config
@@ -43,12 +43,10 @@ microk8s config > ~/.kube/config
 Use the following tox commands to run tests:
 
 ```bash
-tox -e fmt                 # update your code according to linting rules
-tox -e lint                # code style
-tox -e unit                # unit tests
-tox -e smoke-integration   # runs a small subset of integration tests that quickly verifies the charm is mostly working as intended.
-tox -e dev-integration     # Tag integration tests with `@pytest.mark.dev' to select tests to run using this command.
-tox                        # runs 'fmt', 'lint', and 'unit' environments
+tox run -e format              # update your code according to linting rules
+tox run -e lint                # code style
+tox run -e unit                # unit tests
+tox                            # runs 'fmt', 'lint', and 'unit' environments
 ```
 
 Integration tests for individual functionality can be found in tox.ini

pyproject.toml

@@ -12,16 +12,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
-markers = [
-    "smoke: quick integration tests that can be run to quickly verify that the charm builds and deploys ok.",
-    "dev: a quick development marker to run selected tests",
-    "standalone: integration tests for standalone charm function",
-    "backend: integration tests for backend relation to postgresql",
-    "scaling: integration tests for pgbouncer scaling",
-    "legacy_relation: integration tests that test legacy client relations",
-    "client_relation: integration tests that test modern client relations",
-    "tls_tests: integration tests that test TLS",
-]
+markers = ["unstable"]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/conftest.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from pytest_operator.plugin import OpsTest
+
+
+@pytest.fixture(scope="module")
+def ops_test(ops_test: OpsTest) -> OpsTest:
+    if os.environ.get("CI") == "true":
+        # Running in GitHub Actions; skip build step
+        # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml)
+        packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"])
+
+        async def build_charm(charm_path, bases_index: int = None) -> Path:
+            for charm in packed_charms:
+                if Path(charm_path) == Path(charm["directory_path"]):
+                    if bases_index is None or bases_index == charm["bases_index"]:
+                        return charm["file_path"]
+            raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}")
+
+        ops_test.build_charm = build_charm
+    return ops_test

tests/integration/helpers/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/relations/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/relations/pgbouncer_provider/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.