Cache charmcraft pack container, skip unstable tests except on schedule

Ported from canonical/mysql-k8s-operator#146

.github/workflows/ci.yaml

@@ -1,7 +1,13 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
 name: Tests
+
 on:
-  workflow_call:
   pull_request:
+  schedule:
+    - cron: '53 0 * * *' # Daily at 00:53 UTC
+  # Triggered on push to branch "main" by .github/workflows/release.yaml
+  workflow_call:
 
 jobs:
   lint:
@@ -10,10 +16,11 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
         run: python3 -m pip install tox
       - name: Run linters
-        run: tox -e lint
+        run: tox run -e lint
 
   unit-test:
     name: Unit tests
@@ -23,92 +30,61 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
-        run: python -m pip install tox
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
+        run: python3 -m pip install tox
       - name: Run tests
-        run: tox -e unit
-
-  integration-test-lxd-charm:
-    name: Integration tests for the charm (lxd)
-    runs-on: ubuntu-22.04
-    needs:
-      - unit-test
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-      - name: Run integration tests
-        run: |
-          # set sysctl values in case the cloudinit-userdata not applied
-          sudo sysctl -w vm.max_map_count=262144
-          sudo sysctl -w vm.swappiness=0
-          sudo sysctl -w net.ipv4.tcp_retries2=5
+        run: tox run -e unit
 
-          tox -e charm-integration
+  build:
+    name: Build charms
+    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1
 
-  integration-test-lxd-tls:
-    name: Integration tests for TLS (lxd)
-    runs-on: ubuntu-22.04
-    needs:
-      - unit-test
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: lxd
-      - name: Run integration tests
-        run: |
-          # set sysctl values in case the cloudinit-userdata not applied
-          sudo sysctl -w vm.max_map_count=262144
-          sudo sysctl -w vm.swappiness=0
-          sudo sysctl -w net.ipv4.tcp_retries2=5
-
-          tox -e tls-integration
-
-# Commented due to failures because of HW related issues of the github runners:
-#  integration-test-lxd-ha:
-#    name: Integration tests for HA (lxd)
-#    runs-on: ubuntu-22.04
-#    needs:
-#      - unit-test
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v3
-#      - name: Setup operator environment
-#        uses: charmed-kubernetes/actions-operator@main
-#        with:
-#          provider: lxd
-#      - name: Run integration tests
-#        run: |
-#          # set sysctl values in case the cloudinit-userdata not applied
-#          sudo sysctl -w vm.max_map_count=262144
-#          sudo sysctl -w vm.swappiness=0
-#          sudo sysctl -w net.ipv4.tcp_retries2=5
-#
-#          tox -e ha-integration
-
-  integration-test-client-relation:
-    name: Integration tests for client relation
-    runs-on: ubuntu-22.04
+  integration-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        tox-environments:
+          - charm-integration
+          - tls-integration
+#          - ha-integration
+          - client-integration
+    name: ${{ matrix.tox-environments }}
     needs:
       - lint
       - unit-test
+      - build
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup operator environment
+        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: lxd
+      - name: Download packed charm(s)
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ needs.build.outputs.artifact-name }}
+      - name: Select tests
+        id: select-tests
+        run: |
+          if [ "${{ github.event_name }}" == "schedule" ]
+          then
+            echo Running unstable and stable tests
+            echo "mark_expression=" >> $GITHUB_OUTPUT
+          else
+            echo Skipping unstable tests
+            echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
+          fi
       - name: Run integration tests
         run: |
           # set sysctl values in case the cloudinit-userdata not applied
           sudo sysctl -w vm.max_map_count=262144
           sudo sysctl -w vm.swappiness=0
           sudo sysctl -w net.ipv4.tcp_retries2=5
-          tox -e client-integration
+        
+          tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}'
+        env:
+          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}

CONTRIBUTING.md

@@ -23,21 +23,21 @@ this operator.
 
 ## Developing
 
-You can use the environments created by `tox` for development:
+You can create an environment for development with `tox`:
 
 ```shell
-tox --notest -e unit
-source .tox/unit/bin/activate
+tox devenv -e integration
+source venv/bin/activate
 ```
 
 ### Testing
 
 ```shell
-tox -e fmt           # update your code according to linting rules
-tox -e lint          # code style
-tox -e unit          # unit tests
-tox -e integration   # integration tests
-tox                  # runs 'lint' and 'unit' environments
+tox run -e format        # update your code according to linting rules
+tox run -e lint          # code style
+tox run -e unit          # unit tests
+tox run -e integration   # integration tests
+tox                      # runs 'lint' and 'unit' environments
 ```
 
 ## Build charm

pyproject.toml

@@ -12,12 +12,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
-markers = [
-    "charm_tests",
-    "tls_tests",
-    "ha_tests",
-    "client_relation: integration tests that test modern client relations",
-]
+markers = ["unstable"]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/conftest.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from pytest_operator.plugin import OpsTest
+
+
+@pytest.fixture
+def ops_test(ops_test: OpsTest) -> OpsTest:
+    if os.environ.get("CI") == "true":
+        # Running in GitHub Actions; skip build step
+        # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml)
+        packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"])
+
+        async def build_charm(charm_path, bases_index: int = None) -> Path:
+            for charm in packed_charms:
+                if Path(charm_path) == Path(charm["directory_path"]):
+                    if bases_index is None or bases_index == charm["bases_index"]:
+                        return charm["file_path"]
+            raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}")
+
+        ops_test.build_charm = build_charm
+    return ops_test

tests/integration/ha/test_horizontal_scaling.py

@@ -33,7 +33,6 @@
 logger = logging.getLogger(__name__)
 
 
-@pytest.mark.ha_tests
 @pytest.mark.abort_on_fail
 @pytest.mark.skip_if_deployed
 async def test_build_and_deploy(ops_test: OpsTest) -> None:
@@ -65,7 +64,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
     )
 
 
-@pytest.mark.ha_tests
 @pytest.mark.abort_on_fail
 async def test_horizontal_scale_up(ops_test: OpsTest) -> None:
     """Tests that new added units to the cluster are discoverable."""
@@ -92,7 +90,6 @@ async def test_horizontal_scale_up(ops_test: OpsTest) -> None:
     assert ClusterTopology.nodes_count_by_role(nodes)["cluster_manager"] == 3
 
 
-@pytest.mark.ha_tests
 @pytest.mark.abort_on_fail
 async def test_safe_scale_down_shards_realloc(ops_test: OpsTest) -> None:
     """Tests the shutdown of a node, and re-allocation of shards to a newly joined unit.
@@ -180,7 +177,6 @@ async def test_safe_scale_down_shards_realloc(ops_test: OpsTest) -> None:
     assert new_shards_per_node.get(-1, 0) == 0
 
 
-@pytest.mark.ha_tests
 @pytest.mark.abort_on_fail
 async def test_safe_scale_down_roles_reassigning(ops_test: OpsTest) -> None:
     """Tests the shutdown of a node with a role requiring the re-balance of the cluster roles.

tests/integration/relations/opensearch_provider/test_opensearch_provider.py

@@ -37,7 +37,6 @@
 
 
 @pytest.mark.abort_on_fail
-@pytest.mark.client_relation
 async def test_database_relation_with_charm_libraries(
     ops_test: OpsTest, application_charm, opensearch_charm
 ):
@@ -73,7 +72,6 @@ async def test_database_relation_with_charm_libraries(
         await ops_test.model.wait_for_idle(timeout=1200, status="active")
 
 
-@pytest.mark.client_relation
 async def test_database_usage(ops_test: OpsTest):
     """Check we can update and delete things."""
     await run_request(
@@ -104,7 +102,6 @@ async def test_database_usage(ops_test: OpsTest):
     )
 
 
-@pytest.mark.client_relation
 async def test_database_bulk_usage(ops_test: OpsTest):
     """Check we can update and delete things using bulk api."""
     bulk_payload = """{ "index" : { "_index": "albums", "_id" : "2" } }
@@ -145,7 +142,6 @@ async def test_database_bulk_usage(ops_test: OpsTest):
     assert set(artists) == {"Herbie Hancock", "Lydian Collective", "Vulfpeck"}
 
 
-@pytest.mark.client_relation
 async def test_database_version(ops_test: OpsTest):
     """Check version is accurate."""
     run_version_request = await run_request(
@@ -166,7 +162,6 @@ async def test_database_version(ops_test: OpsTest):
     assert version == results.get("version", {}).get("number")
 
 
-@pytest.mark.client_relation
 async def test_multiple_relations(ops_test: OpsTest, application_charm):
     """Test that two different applications can connect to the database."""
     # Deploy secondary application.
@@ -194,7 +189,6 @@ async def test_multiple_relations(ops_test: OpsTest, application_charm):
         )
 
 
-@pytest.mark.client_relation
 async def test_relation_broken(ops_test: OpsTest):
     """Test that the user is removed when the relation is broken."""
     # Retrieve the relation user.

tests/integration/test_charm.py

@@ -26,7 +26,6 @@
 DEFAULT_NUM_UNITS = 2
 
 
-@pytest.mark.charm_tests
 @pytest.mark.abort_on_fail
 @pytest.mark.skip_if_deployed
 async def test_build_and_deploy(ops_test: OpsTest) -> None:
@@ -42,7 +41,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
     await ops_test.model.wait_for_idle(wait_for_exact_units=DEFAULT_NUM_UNITS)
 
 
-@pytest.mark.charm_tests
 @pytest.mark.abort_on_fail
 async def test_status(ops_test: OpsTest) -> None:
     """Verifies that the application and unit are active."""
@@ -52,7 +50,6 @@ async def test_status(ops_test: OpsTest) -> None:
     assert len(ops_test.model.applications[APP_NAME].units) == DEFAULT_NUM_UNITS
 
 
-@pytest.mark.charm_tests
 @pytest.mark.abort_on_fail
 async def test_actions_get_admin_password(ops_test: OpsTest) -> None:
     """Test the retrieval of admin secrets."""
@@ -91,7 +88,6 @@ async def test_actions_get_admin_password(ops_test: OpsTest) -> None:
     assert result.status == "failed"
 
 
-@pytest.mark.charm_tests
 @pytest.mark.abort_on_fail
 async def test_actions_rotate_admin_password(ops_test: OpsTest) -> None:
     """Test the rotation and change of admin password."""

tests/integration/tls/test_tls.py

@@ -27,7 +27,6 @@
 TLS_CERTIFICATES_APP_NAME = "tls-certificates-operator"
 
 
-@pytest.mark.tls_tests
 @pytest.mark.abort_on_fail
 @pytest.mark.skip_if_deployed
 async def test_build_and_deploy_active(ops_test: OpsTest) -> None:
@@ -57,7 +56,6 @@ async def test_build_and_deploy_active(ops_test: OpsTest) -> None:
     )
 
 
-@pytest.mark.tls_tests
 @pytest.mark.abort_on_fail
 async def test_security_index_initialised(ops_test: OpsTest) -> None:
     """Test that the security index is well initialised."""
@@ -66,15 +64,13 @@ async def test_security_index_initialised(ops_test: OpsTest) -> None:
     assert await check_security_index_initialised(ops_test, leader_unit_ip)
 
 
-@pytest.mark.tls_tests
 @pytest.mark.abort_on_fail
 async def test_tls_configured(ops_test: OpsTest) -> None:
     """Test that TLS is enabled when relating to the TLS Certificates Operator."""
     for unit_name, unit_ip in get_application_unit_ips_names(ops_test).items():
         assert await check_unit_tls_configured(ops_test, unit_ip, unit_name)
 
 
-@pytest.mark.tls_tests
 @pytest.mark.abort_on_fail
 async def test_cluster_formation_after_tls(ops_test: OpsTest) -> None:
     """Test that the cluster formation is successful after TLS setup."""