From eab4bd19d894a776ee6894f0ebe64a93d8b6c6d0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 17 Feb 2023 21:13:26 +0000 Subject: [PATCH] Cache charmcraft pack container, skip unstable tests except on schedule Ported from https://github.com/canonical/mysql-k8s-operator/pull/146 --- .github/workflows/ci.yaml | 102 ++++++++++++-------------- CONTRIBUTING.md | 16 ++-- pyproject.toml | 1 + tests/integration/__init__.py | 2 + tests/integration/conftest.py | 21 ++++++ tests/integration/test_charm.py | 5 +- tests/integration/test_kafka_charm.py | 2 +- tests/integration/test_s3_charm.py | 2 +- tests/unit/__init__.py | 2 + tox.ini | 52 ++++++------- 10 files changed, 108 insertions(+), 97 deletions(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/unit/__init__.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e18493f3..c400246a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,12 @@ -name: PR Tests +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +name: Tests + on: pull_request: + schedule: + - cron: '53 0 * * *' # Daily at 00:53 UTC + # Triggered on push to branch "main" by .github/workflows/release.yaml workflow_call: jobs: @@ -10,10 +16,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Install dependencies + - name: Install tox + # TODO: Consider replacing with custom image on self-hosted runner OR pinning version run: python3 -m pip install tox - name: Run linters - run: tox -e lint + run: tox run -e lint unit-test: name: Unit tests @@ -21,80 +28,61 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Install dependencies - run: python -m pip install tox + - name: Install tox + # TODO: Consider replacing with custom image on self-hosted runner OR pinning version + run: python3 -m pip install tox - name: Run tests - run: tox -e unit + run: tox run -e unit - integration-test-database: - name: Integration tests database library (microk8s) - needs: - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - bootstrap-options: "--agent-version 2.9.38" - - name: Run integration tests - # set a predictable model name so it can be consumed by charm-logdump-action - run: tox -e integration-database -- --model testing - - name: Dump logs - uses: canonical/charm-logdump-action@main - if: failure() - with: - app: data-platform-libs - model: testing - - integration-test-kafka: - name: Integration tests kafka library (microk8s) - needs: - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - bootstrap-options: "--agent-version 2.9.38" - - name: Run integration tests - # set a predictable model name so it can be consumed by charm-logdump-action - run: tox -e integration-kafka -- --model testing - - name: Dump logs - uses: canonical/charm-logdump-action@main - if: failure() - with: - app: data-platform-libs - model: testing + build: + name: Build charms + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1 - integration-test-s3: - name: Integration tests s3 library (microk8s) + integration-test: + strategy: + fail-fast: false + matrix: + tox-environments: + - integration-database + - integration-s3 + name: ${{ matrix.tox-environments }} needs: - lint - unit-test + - build runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Setup operator environment + # TODO: Replace with custom image on self-hosted runner uses: charmed-kubernetes/actions-operator@main with: provider: microk8s bootstrap-options: "--agent-version 2.9.38" + - name: Download packed charm(s) + uses: actions/download-artifact@v3 + with: + name: ${{ needs.build.outputs.artifact-name }} + - name: Select tests + id: select-tests + run: | + if [ "${{ github.event_name }}" == "schedule" ] + then + echo Running unstable and stable tests + echo "mark_expression=" >> $GITHUB_OUTPUT + else + echo Skipping unstable tests + echo "mark_expression=not unstable" >> $GITHUB_OUTPUT + fi - name: Run integration tests # set a predictable model name so it can be consumed by charm-logdump-action - run: tox -e integration-s3 -- --model testing + run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing + env: + CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} - name: Dump logs uses: canonical/charm-logdump-action@main if: failure() with: app: data-platform-libs model: testing - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8bd556e5..d338911a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,21 +24,21 @@ this operator. ## Developing -You can use the environments created by `tox` for development: +You can create an environment for development with `tox`: ```shell -tox --notest -e unit -source .tox/unit/bin/activate +tox devenv -e integration +source venv/bin/activate ``` ### Testing ```shell -tox -e fmt # update your code according to linting rules -tox -e lint # code style -tox -e unit # unit tests -tox -e integration # integration tests -tox # runs 'lint' and 'unit' environments +tox run -e format # update your code according to linting rules +tox run -e lint # code style +tox run -e unit # unit tests +tox run -e integration # integration tests +tox # runs 'lint' and 'unit' environments ``` ## Build charm diff --git a/pyproject.toml b/pyproject.toml index 39befc12..4e9d36cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ show_missing = true minversion = "6.0" log_cli_level = "INFO" asyncio_mode = "auto" +markers = ["unstable"] # Formatting tools configuration [tool.black] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..db3bfe1a --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d0f3d88e..d28b288f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,12 +2,33 @@ # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. +import json +import os import shutil +from pathlib import Path import pytest from pytest_operator.plugin import OpsTest +@pytest.fixture +def ops_test(ops_test: OpsTest) -> OpsTest: + if os.environ.get("CI") == "true": + # Running in GitHub Actions; skip build step + # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml) + packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"]) + + async def build_charm(charm_path, bases_index: int = None) -> Path: + for charm in packed_charms: + if Path(charm_path) == Path(charm["directory_path"]): + if bases_index is None or bases_index == charm["bases_index"]: + return charm["file_path"] + raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}") + + ops_test.build_charm = build_charm + return ops_test + + @pytest.fixture(scope="module", autouse=True) def copy_data_interfaces_library_into_charm(ops_test: OpsTest): """Copy the data_interfaces library to the different charm folder.""" diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 2421fe73..9ada73d0 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -10,10 +10,7 @@ import yaml from pytest_operator.plugin import OpsTest -from tests.integration.helpers import ( - build_connection_string, - get_application_relation_data, -) +from .helpers import build_connection_string, get_application_relation_data logger = logging.getLogger(__name__) diff --git a/tests/integration/test_kafka_charm.py b/tests/integration/test_kafka_charm.py index 2ce19a82..e54e3352 100644 --- a/tests/integration/test_kafka_charm.py +++ b/tests/integration/test_kafka_charm.py @@ -7,7 +7,7 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration.helpers import get_application_relation_data +from .helpers import get_application_relation_data logger = logging.getLogger(__name__) diff --git a/tests/integration/test_s3_charm.py b/tests/integration/test_s3_charm.py index f58a3b3d..e015705d 100644 --- a/tests/integration/test_s3_charm.py +++ b/tests/integration/test_s3_charm.py @@ -7,7 +7,7 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration.helpers import get_connection_info +from .helpers import get_connection_info logger = logging.getLogger(__name__) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..db3bfe1a --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tox.ini b/tox.ini index 5b23fc95..b0c8e4e7 100644 --- a/tox.ini +++ b/tox.ini @@ -2,27 +2,27 @@ # See LICENSE file for licensing details. [tox] -skipsdist=True +no_package = True skip_missing_interpreters = True -envlist = lint, unit +env_list = lint, unit [vars] -src_path = {toxinidir}/src/ -tst_path = {toxinidir}/tests/ -lib_path = {toxinidir}/lib/charms/data_platform_libs -all_path = {[vars]src_path} {[vars]tst_path} {[vars]lib_path} +src_path = {tox_root}/src +tests_path = {tox_root}/tests +lib_path = {tox_root}/lib/charms/data_platform_libs +all_path = {[vars]src_path} {[vars]tests_path} {[vars]lib_path} [testenv] -setenv = - PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path} - PYTHONBREAKPOINT=ipdb.set_trace - PY_COLORS=1 -passenv = - PYTHONPATH - CHARM_BUILD_DIR - MODEL_SETTINGS +set_env = + PYTHONPATH = {tox_root}/lib:{[vars]src_path} + PYTHONBREAKPOINT=ipdb.set_trace + PY_COLORS=1 +pass_env = + PYTHONPATH + CHARM_BUILD_DIR + MODEL_SETTINGS -[testenv:fmt] +[testenv:format] description = Apply coding style standards to code deps = black @@ -45,9 +45,9 @@ deps = codespell commands = codespell {[vars]lib_path} - codespell {toxinidir} --skip {toxinidir}/.git --skip {toxinidir}/.tox \ - --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/venv \ - --skip {toxinidir}/.mypy_cache --skip {toxinidir}/icon.svg + codespell {tox_root} --skip {tox_root}/.git --skip {tox_root}/.tox \ + --skip {tox_root}/build --skip {tox_root}/lib --skip {tox_root}/venv \ + --skip {tox_root}/.mypy_cache --skip {tox_root}/icon.svg # pflake8 wrapper supports config from pyproject.toml pflake8 {[vars]all_path} isort --check-only --diff {[vars]all_path} @@ -59,10 +59,10 @@ deps = parameterized pytest coverage[toml] - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = coverage run --source={[vars]src_path},{[vars]lib_path} \ - -m pytest --ignore={[vars]tst_path}integration -v --tb native -s {posargs} + -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit coverage report [testenv:integration-database] @@ -72,9 +72,9 @@ deps = pytest juju==2.9.38.1 pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} {[vars]tst_path}integration/test_charm.py + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_charm.py [testenv:integration-kafka] description = Run integration tests @@ -83,9 +83,9 @@ deps = pytest juju==2.9.38.1 pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} {[vars]tst_path}integration/test_kafka_charm.py + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_kafka_charm.py [testenv:integration-s3] description = Run integration tests @@ -94,6 +94,6 @@ deps = pytest juju==2.9.38.1 pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} {[vars]tst_path}integration/test_s3_charm.py + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_s3_charm.py