Cache charmcraft pack container, skip unstable tests except on schedule

Ported from canonical/mysql-k8s-operator#146

.github/workflows/ci.yaml

@@ -1,6 +1,12 @@
-name: PR Tests
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+name: Tests
+
 on:
   pull_request:
+  schedule:
+    - cron: '53 0 * * *' # Daily at 00:53 UTC
+  # Triggered on push to branch "main" by .github/workflows/release.yaml
   workflow_call:
 
 jobs:
@@ -10,91 +16,73 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
         run: python3 -m pip install tox
       - name: Run linters
-        run: tox -e lint
+        run: tox run -e lint
 
   unit-test:
     name: Unit tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Install dependencies
-        run: python -m pip install tox
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
+        run: python3 -m pip install tox
       - name: Run tests
-        run: tox -e unit
+        run: tox run -e unit
 
-  integration-test-database:
-    name: Integration tests database library (microk8s)
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          bootstrap-options: "--agent-version 2.9.38"
-      - name: Run integration tests
-        # set a predictable model name so it can be consumed by charm-logdump-action
-        run: tox -e integration-database -- --model testing
-      - name: Dump logs
-        uses: canonical/charm-logdump-action@main
-        if: failure()
-        with:
-          app: data-platform-libs
-          model: testing
-
-  integration-test-kafka:
-    name: Integration tests kafka library (microk8s)
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          bootstrap-options: "--agent-version 2.9.38"
-      - name: Run integration tests
-        # set a predictable model name so it can be consumed by charm-logdump-action
-        run: tox -e integration-kafka -- --model testing
-      - name: Dump logs
-        uses: canonical/charm-logdump-action@main
-        if: failure()
-        with:
-          app: data-platform-libs
-          model: testing
+  build:
+    name: Build charms
+    uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1
 
-  integration-test-s3:
-    name: Integration tests s3 library (microk8s)
+  integration-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        tox-environments:
+          - integration-database
+          - integration-s3
+    name: ${{ matrix.tox-environments }}
     needs:
       - lint
       - unit-test
+      - build
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup operator environment
+        # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
         with:
           provider: microk8s
           bootstrap-options: "--agent-version 2.9.38"
+      - name: Download packed charm(s)
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ needs.build.outputs.artifact-name }}
+      - name: Select tests
+        id: select-tests
+        run: |
+          if [ "${{ github.event_name }}" == "schedule" ]
+          then
+            echo Running unstable and stable tests
+            echo "mark_expression=" >> $GITHUB_OUTPUT
+          else
+            echo Skipping unstable tests
+            echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
+          fi
       - name: Run integration tests
         # set a predictable model name so it can be consumed by charm-logdump-action
-        run: tox -e integration-s3 -- --model testing
+        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing
+        env:
+          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
       - name: Dump logs
         uses: canonical/charm-logdump-action@main
         if: failure()
         with:
           app: data-platform-libs
           model: testing
-

CONTRIBUTING.md

@@ -24,21 +24,21 @@ this operator.
 
 ## Developing
 
-You can use the environments created by `tox` for development:
+You can create an environment for development with `tox`:
 
 ```shell
-tox --notest -e unit
-source .tox/unit/bin/activate
+tox devenv -e integration
+source venv/bin/activate
 ```
 
 ### Testing
 
 ```shell
-tox -e fmt           # update your code according to linting rules
-tox -e lint          # code style
-tox -e unit          # unit tests
-tox -e integration   # integration tests
-tox                  # runs 'lint' and 'unit' environments
+tox run -e format        # update your code according to linting rules
+tox run -e lint          # code style
+tox run -e unit          # unit tests
+tox run -e integration   # integration tests
+tox                      # runs 'lint' and 'unit' environments
 ```
 
 ## Build charm

pyproject.toml

@@ -12,6 +12,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
+markers = ["unstable"]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/conftest.py

@@ -2,12 +2,33 @@
 # Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
 
+import json
+import os
 import shutil
+from pathlib import Path
 
 import pytest
 from pytest_operator.plugin import OpsTest
 
 
+@pytest.fixture
+def ops_test(ops_test: OpsTest) -> OpsTest:
+    if os.environ.get("CI") == "true":
+        # Running in GitHub Actions; skip build step
+        # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml)
+        packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"])
+
+        async def build_charm(charm_path, bases_index: int = None) -> Path:
+            for charm in packed_charms:
+                if Path(charm_path) == Path(charm["directory_path"]):
+                    if bases_index is None or bases_index == charm["bases_index"]:
+                        return charm["file_path"]
+            raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}")
+
+        ops_test.build_charm = build_charm
+    return ops_test
+
+
 @pytest.fixture(scope="module", autouse=True)
 def copy_data_interfaces_library_into_charm(ops_test: OpsTest):
     """Copy the data_interfaces library to the different charm folder."""

tests/integration/test_charm.py

@@ -10,10 +10,7 @@
 import yaml
 from pytest_operator.plugin import OpsTest
 
-from tests.integration.helpers import (
-    build_connection_string,
-    get_application_relation_data,
-)
+from .helpers import build_connection_string, get_application_relation_data
 
 logger = logging.getLogger(__name__)
 

tests/integration/test_kafka_charm.py

@@ -7,7 +7,7 @@
 import pytest
 from pytest_operator.plugin import OpsTest
 
-from tests.integration.helpers import get_application_relation_data
+from .helpers import get_application_relation_data
 
 logger = logging.getLogger(__name__)
 

tests/integration/test_s3_charm.py

@@ -7,7 +7,7 @@
 import pytest
 from pytest_operator.plugin import OpsTest
 
-from tests.integration.helpers import get_connection_info
+from .helpers import get_connection_info
 
 logger = logging.getLogger(__name__)
 

tests/unit/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tox.ini

@@ -2,27 +2,27 @@
 # See LICENSE file for licensing details.
 
 [tox]
-skipsdist=True
+no_package = True
 skip_missing_interpreters = True
-envlist = lint, unit
+env_list = lint, unit
 
 [vars]
-src_path = {toxinidir}/src/
-tst_path = {toxinidir}/tests/
-lib_path = {toxinidir}/lib/charms/data_platform_libs
-all_path = {[vars]src_path} {[vars]tst_path} {[vars]lib_path}
+src_path = {tox_root}/src
+tests_path = {tox_root}/tests
+lib_path = {tox_root}/lib/charms/data_platform_libs
+all_path = {[vars]src_path} {[vars]tests_path} {[vars]lib_path}
 
 [testenv]
-setenv =
-  PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path}
-  PYTHONBREAKPOINT=ipdb.set_trace
-  PY_COLORS=1
-passenv =
-  PYTHONPATH
-  CHARM_BUILD_DIR
-  MODEL_SETTINGS
+set_env =
+    PYTHONPATH = {tox_root}/lib:{[vars]src_path}
+    PYTHONBREAKPOINT=ipdb.set_trace
+    PY_COLORS=1
+pass_env =
+    PYTHONPATH
+    CHARM_BUILD_DIR
+    MODEL_SETTINGS
 
-[testenv:fmt]
+[testenv:format]
 description = Apply coding style standards to code
 deps =
     black
@@ -45,9 +45,9 @@ deps =
     codespell
 commands =
     codespell {[vars]lib_path}
-    codespell {toxinidir} --skip {toxinidir}/.git --skip {toxinidir}/.tox \
-      --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/venv \
-      --skip {toxinidir}/.mypy_cache --skip {toxinidir}/icon.svg
+    codespell {tox_root} --skip {tox_root}/.git --skip {tox_root}/.tox \
+      --skip {tox_root}/build --skip {tox_root}/lib --skip {tox_root}/venv \
+      --skip {tox_root}/.mypy_cache --skip {tox_root}/icon.svg
     # pflake8 wrapper supports config from pyproject.toml
     pflake8 {[vars]all_path}
     isort --check-only --diff {[vars]all_path}
@@ -59,10 +59,10 @@ deps =
     parameterized
     pytest
     coverage[toml]
-    -r{toxinidir}/requirements.txt
+    -r {tox_root}/requirements.txt
 commands =
     coverage run --source={[vars]src_path},{[vars]lib_path} \
-        -m pytest --ignore={[vars]tst_path}integration -v --tb native -s {posargs}
+        -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit
     coverage report
 
 [testenv:integration-database]
@@ -72,9 +72,9 @@ deps =
     pytest
     juju==2.9.38.1
     pytest-operator
-    -r{toxinidir}/requirements.txt
+    -r {tox_root}/requirements.txt
 commands =
-    pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} {[vars]tst_path}integration/test_charm.py
+    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_charm.py
 
 [testenv:integration-kafka]
 description = Run integration tests
@@ -83,9 +83,9 @@ deps =
     pytest
     juju==2.9.38.1
     pytest-operator
-    -r{toxinidir}/requirements.txt
+    -r {tox_root}/requirements.txt
 commands =
-    pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} {[vars]tst_path}integration/test_kafka_charm.py
+    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_kafka_charm.py
 
 [testenv:integration-s3]
 description = Run integration tests
@@ -94,6 +94,6 @@ deps =
     pytest
     juju==2.9.38.1
     pytest-operator
-    -r{toxinidir}/requirements.txt
+    -r {tox_root}/requirements.txt
 commands =
-    pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} {[vars]tst_path}integration/test_s3_charm.py
+    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_s3_charm.py