Cache charmcraft pack container, skip unstable tests except on schedule

.github/workflows/ci.yaml

@@ -1,8 +1,13 @@
-name: integration and other tests
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+name: Tests
 
 on:
-  workflow_call:
   pull_request:
+  schedule:
+    - cron: '53 0 * * *' # Daily at 00:53 UTC
+  # Triggered on push to branch "main" by .github/workflows/release.yaml
+  workflow_call:
 
 jobs:
   lint:
@@ -11,127 +16,71 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
         run: python3 -m pip install tox
       - name: Run linters
-        run: tox -e lint
+        run: tox run -e lint
 
   unit-test:
     name: Unit tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
-        run: python -m pip install tox
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
+        run: python3 -m pip install tox
       - name: Run tests
-        run: tox -e unit
-
-  integration-test-microk8s-charm:
-    name: Charm integration tests (microk8s)
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Run charm integration tests
-        run: tox -e integration-charm
-
-  integration-test-database-relation:
-    name: Integration tests for database relation
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Run integration database tests
-        run: tox -e integration-database-relation
+        run: tox run -e unit
 
-  integration-test-microk8s-osm-mysql:
-    name: Integration osm-msyql relation tests (microk8s)
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Run osm-mysql relation integration tests
-        run: tox -e integration-osm-mysql
+  build:
+    name: Build charms
+    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1
 
-  integration-test-microk8s-replication:
-    name: Integration replication tests (microk8s)
+  integration-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        tox-environments:
+          - integration-charm
+          - integration-database-relation
+          - integration-osm-mysql
+          - integration-replication
+          - integration-self-healing
+          - integration-tls
+    name: ${{ matrix.tox-environments }}
     needs:
       - lint
       - unit-test
+      - build
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup operator environment
+        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: microk8s
           # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed
           bootstrap-options: "--agent-version 2.9.29"
-      - name: Run replication integration tests
-        run: tox -e integration-replication
-
-# TODO: re-enable when stable (currently flaky)
-#  integration-test-microk8s-self-healing:
-#    name: Integration self healing tests (microk8s)
-#    needs:
-#      - lint
-#      - unit-test
-#    runs-on: ubuntu-latest
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v3
-#      - name: Setup operator environment
-#        uses: charmed-kubernetes/actions-operator@main
-#        with:
-#          provider: microk8s
-#          microk8s-group: microk8s
-#          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed
-#          bootstrap-options: "--agent-version 2.9.29"
-#      - name: Run self healing integration tests
-#        run: tox -e integration-self-healing
-
-  integration-test-tls:
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
+      - name: Download packed charm(s)
+        uses: actions/download-artifact@v3
         with:
-          provider: microk8s
-          # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed
-          bootstrap-options: "--agent-version 2.9.29"
-      - name: Integration tls relation tests
-        run: tox -e integration-tls
+          name: ${{ needs.build.outputs.artifact-name }}
+      - name: Select tests
+        id: select-tests
+        run: |
+          if [ "${{ github.event_name }}" == "schedule" ]
+          then
+            echo Running unstable and stable tests
+            echo "mark_expression=" >> $GITHUB_OUTPUT
+          else
+            echo Skipping unstable tests
+            echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
+          fi
+      - name: Run integration tests
+        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}'
+        env:
+          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}

CONTRIBUTING.md

@@ -21,21 +21,21 @@ this operator.
 
 ## Developing
 
-You can use the environments created by `tox` for development:
+You can create an environment for development with `tox`:
 
 ```shell
-tox --notest -e unit
-source .tox/unit/bin/activate
+tox devenv -e integration
+source venv/bin/activate
 ```
 
 ### Testing
 
 ```shell
-tox -e fmt             # update your code according to linting rules
-tox -e lint            # code style
-tox -e unit            # unit tests
-tox -e integration-*   # integration tests
-tox                    # runs 'lint' and 'unit' environments
+tox run -e format          # update your code according to linting rules
+tox run -e lint            # code style
+tox run -e unit            # unit tests
+tox run -e integration-*   # integration tests
+tox                        # runs 'lint' and 'unit' environments
 ```
 
 ## Build charm

pyproject.toml

@@ -12,14 +12,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
-markers = [
-    "charm_tests",
-    "database_tests",
-    "osm_mysql_tests",
-    "replication_tests",
-    "self_healing_tests",
-    "tls_tests",
-]
+markers = ["unstable"]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/conftest.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from pytest_operator.plugin import OpsTest
+
+
+@pytest.fixture
+def ops_test(ops_test: OpsTest) -> OpsTest:
+    if os.environ.get("CI") == "true":
+        # Running in GitHub Actions; skip build step
+        # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml)
+        packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"])
+
+        async def build_charm(charm_path, bases_index: int = None) -> Path:
+            for charm in packed_charms:
+                if Path(charm_path) == Path(charm["directory_path"]):
+                    if bases_index is None or bases_index == charm["bases_index"]:
+                        return charm["file_path"]
+            raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}")
+
+        ops_test.build_charm = build_charm
+    return ops_test

tests/integration/helpers.py

@@ -20,7 +20,8 @@
 from tenacity import retry, stop_after_attempt, wait_fixed
 
 from constants import SERVER_CONFIG_USERNAME
-from tests.integration.connector import MySQLConnector
+
+from .connector import MySQLConnector
 
 
 def generate_random_string(length: int) -> str:

tests/integration/high_availability/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/high_availability/conftest.py

@@ -8,7 +8,8 @@
 from pytest_operator.plugin import OpsTest
 
 from constants import CONTAINER_NAME, MYSQLD_SERVICE
-from tests.integration.high_availability.high_availability_helpers import (
+
+from .high_availability_helpers import (
     deploy_chaos_mesh,
     destroy_chaos_mesh,
     get_application_name,

tests/integration/high_availability/high_availability_helpers.py

@@ -24,7 +24,7 @@
     wait_fixed,
 )
 
-from tests.integration.helpers import (
+from ..helpers import (
     execute_queries_on_unit,
     generate_random_string,
     get_cluster_status,

tests/integration/high_availability/test_replication.py

@@ -11,14 +11,14 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import Retrying, stop_after_delay, wait_fixed
 
-from tests.integration.helpers import (
+from ..helpers import (
     execute_queries_on_unit,
     get_primary_unit,
     get_server_config_credentials,
     get_unit_address,
     scale_application,
 )
-from tests.integration.high_availability.high_availability_helpers import (
+from .high_availability_helpers import (
     clean_up_database_and_table,
     deploy_and_scale_mysql,
     ensure_all_units_continuous_writes_incrementing,
@@ -32,14 +32,12 @@
 TIMEOUT = 15 * 60
 
 
-@pytest.mark.replication_tests
 async def test_build_and_deploy(ops_test: OpsTest) -> None:
     """Simple test to ensure that the mysql and application charms get deployed."""
     await high_availability_test_setup(ops_test)
 
 
 @pytest.mark.abort_on_fail
-@pytest.mark.replication_tests
 async def test_check_consistency(ops_test: OpsTest, continuous_writes) -> None:
     """Test to write to primary, and read the same data back from replicas."""
     mysql_application_name, _ = await high_availability_test_setup(ops_test)
@@ -55,7 +53,6 @@ async def test_check_consistency(ops_test: OpsTest, continuous_writes) -> None:
 
 
 @pytest.mark.abort_on_fail
-@pytest.mark.replication_tests
 async def test_no_replication_across_clusters(ops_test: OpsTest, continuous_writes) -> None:
     """Test to ensure that writes to one cluster do not replicate to another cluster."""
     mysql_application_name, _ = await high_availability_test_setup(ops_test)
@@ -112,7 +109,6 @@ async def test_no_replication_across_clusters(ops_test: OpsTest, continuous_writ
 
 
 @pytest.mark.abort_on_fail
-@pytest.mark.replication_tests
 async def test_scaling_without_data_loss(ops_test: OpsTest) -> None:
     """Test to ensure that data is preserved when a unit is scaled up and then down.
 
@@ -193,7 +189,6 @@ async def test_scaling_without_data_loss(ops_test: OpsTest) -> None:
 # TODO: move test immediately after "test_build_and_deploy" once the following issue is resolved
 # https://github.com/canonical/mysql-k8s-operator/issues/102
 @pytest.mark.abort_on_fail
-@pytest.mark.replication_tests
 async def test_kill_primary_check_reelection(ops_test: OpsTest, continuous_writes) -> None:
     """Test to kill the primary under load and ensure re-election of primary."""
     mysql_application_name, _ = await high_availability_test_setup(ops_test)