From 725d26ec9dfac5d4ad3191f132cfce248402c34a Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 17 Feb 2023 20:42:08 +0000 Subject: [PATCH 1/3] Cache charmcraft pack container, skip unstable tests except on schedule Ported from https://github.com/canonical/mysql-k8s-operator/pull/146 --- .github/workflows/ci.yaml | 109 +++++++++----------- .github/workflows/release.yaml | 109 +------------------- CONTRIBUTING.md | 16 +-- pyproject.toml | 1 + tests/__init__.py | 6 -- tests/integration/conftest.py | 28 +++++ tests/integration/test_password_rotation.py | 4 +- tests/integration/test_provider.py | 4 +- tests/integration/test_scaling.py | 4 +- tests/integration/test_tls.py | 4 +- tests/unit/__init__.py | 2 + tox.ini | 74 ++++++++----- 12 files changed, 149 insertions(+), 212 deletions(-) delete mode 100644 tests/__init__.py create mode 100644 tests/integration/conftest.py create mode 100644 tests/unit/__init__.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 05817995..5fdce683 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,13 @@ -name: Testing +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +name: Tests + on: pull_request: - workflow_dispatch: + schedule: + - cron: '53 0 * * *' # Daily at 00:53 UTC + # Triggered on push to branch "main" by .github/workflows/release.yaml + workflow_call: jobs: lint: @@ -9,83 +15,68 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 - - name: Install dependencies + uses: actions/checkout@v3 + - name: Install tox + # TODO: Consider replacing with custom image on self-hosted runner OR pinning version run: python3 -m pip install tox - name: Run linters - run: tox -e lint + run: tox run -e lint unit-test: name: Unit tests runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 - - name: Install dependencies - run: python -m pip install tox + uses: actions/checkout@v3 + - name: Install tox + # TODO: Consider replacing with custom image on self-hosted runner OR pinning version + run: python3 -m pip install tox - name: Run tests - run: tox -e unit - - integration-test-password-rotation: - name: Integration tests for password-rotation (microk8s) - needs: - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - - name: Run integration tests - run: tox -e integration-password-rotation + run: tox run -e unit - integration-test-provider: - name: Integration tests for provider (microk8s) - needs: - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - - name: Run integration tests - run: tox -e integration-provider + build: + name: Build charms + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1 - integration-test-scaling: - name: Integration tests for scaling (microk8s) + integration-test: + strategy: + fail-fast: false + matrix: + tox-environments: + - integration-password-rotation + - integration-provider + - integration-scaling + - integration-tls + name: ${{ matrix.tox-environments }} needs: - lint - unit-test + - build runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup operator environment + # TODO: Replace with custom image on self-hosted runner uses: charmed-kubernetes/actions-operator@main with: provider: microk8s - - name: Run integration tests - run: tox -e integration-scaling - - integration-test-tls: - name: Integration tests for tls (microk8s) - needs: - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main + - name: Download packed charm(s) + uses: actions/download-artifact@v3 with: - provider: microk8s + name: ${{ needs.build.outputs.artifact-name }} + - name: Select tests + id: select-tests + run: | + if [ "${{ github.event_name }}" == "schedule" ] + then + echo Running unstable and stable tests + echo "mark_expression=" >> $GITHUB_OUTPUT + else + echo Skipping unstable tests + echo "mark_expression=not unstable" >> $GITHUB_OUTPUT + fi - name: Run integration tests - run: tox -e integration-tls + run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' + env: + CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 06012049..098c3e25 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,3 @@ - name: Release to latest/edge on: @@ -12,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Check libs @@ -21,118 +20,20 @@ jobs: credentials: "${{ secrets.CHARMHUB_TOKEN }}" # FIXME: current token will expire in 2023-07-04 github-token: "${{ secrets.GITHUB_TOKEN }}" - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install dependencies - run: python3 -m pip install tox - - name: Run linters - run: tox -e lint - - unit-test: - name: Unit tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install dependencies - run: python -m pip install tox - - name: Run tests - run: tox -e unit - - integration-test-password-rotation: - name: Integration tests for password-rotation (microk8s) - needs: - - lib-check - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed - bootstrap-options: "--agent-version 2.9.29" - - name: Run integration tests - run: tox -e integration-password-rotation - - integration-test-provider: - name: Integration tests for provider (microk8s) - needs: - - lib-check - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed - bootstrap-options: "--agent-version 2.9.29" - - name: Run integration tests - run: tox -e integration-provider - - integration-test-scaling: - name: Integration tests for scaling (microk8s) + ci-tests: needs: - lib-check - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed - bootstrap-options: "--agent-version 2.9.29" - - name: Run integration tests - run: tox -e integration-scaling - - integration-test-tls: - name: Integration tests for tls (microk8s) - needs: - - lib-check - - lint - - unit-test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: microk8s - # This is needed until https://bugs.launchpad.net/juju/+bug/1977582 is fixed - bootstrap-options: "--agent-version 2.9.29" - - name: Run integration tests - run: tox -e integration-tls + uses: ./.github/workflows/ci.yaml release-to-charmhub: name: Release to CharmHub needs: - lib-check - - lint - - unit-test - - integration-test-scaling - - integration-test-provider - - integration-test-password-rotation - - integration-test-tls + - ci-tests runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Select charmhub channel diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad9ab0ca..99e77b15 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,21 +21,21 @@ To run the charm locally with Juju, it is recommended to use [LXD](https://linux ## Developing -You can use the environments created by `tox` for development: +You can create an environment for development with `tox`: ```shell -tox --notest -e unit -source .tox/unit/bin/activate +tox devenv -e integration +source venv/bin/activate ``` ### Testing ```shell -tox -e fmt # update your code according to linting rules -tox -e lint # code style -tox -e unit # unit tests -tox -e integration # integration tests -tox # runs 'lint' and 'unit' environments +tox run -e format # update your code according to linting rules +tox run -e lint # code style +tox run -e unit # unit tests +tox run -e integration # integration tests +tox # runs 'lint' and 'unit' environments ``` ## Build and Deploy diff --git a/pyproject.toml b/pyproject.toml index 747a041d..db4a83ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ show_missing = true [tool.pytest.ini_options] minversion = "6.0" log_cli_level = "INFO" +markers = ["unstable"] # Formatting tools configuration [tool.black] diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 6f5ac12c..00000000 --- a/tests/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. -import ops.testing - -ops.testing.SIMULATE_CAN_CONNECT = True diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000..a5a9e365 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. + +import json +import os +from pathlib import Path + +import pytest +from pytest_operator.plugin import OpsTest + + +@pytest.fixture +def ops_test(ops_test: OpsTest) -> OpsTest: + if os.environ.get("CI") == "true": + # Running in GitHub Actions; skip build step + # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml) + packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"]) + + async def build_charm(charm_path, bases_index: int = None) -> Path: + for charm in packed_charms: + if Path(charm_path) == Path(charm["directory_path"]): + if bases_index is None or bases_index == charm["bases_index"]: + return charm["file_path"] + raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}") + + ops_test.build_charm = build_charm + return ops_test diff --git a/tests/integration/test_password_rotation.py b/tests/integration/test_password_rotation.py index f35e512e..98f807cf 100644 --- a/tests/integration/test_password_rotation.py +++ b/tests/integration/test_password_rotation.py @@ -7,8 +7,8 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration import APP_NAME, SERIES, ZOOKEEPER_IMAGE -from tests.integration.helpers import ( +from . import APP_NAME, SERIES, ZOOKEEPER_IMAGE +from .helpers import ( check_key, get_address, get_user_password, diff --git a/tests/integration/test_provider.py b/tests/integration/test_provider.py index bfbeb46b..54c21098 100644 --- a/tests/integration/test_provider.py +++ b/tests/integration/test_provider.py @@ -8,8 +8,8 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration import SERIES, ZOOKEEPER_IMAGE -from tests.integration.helpers import check_jaas_config, ping_servers +from . import SERIES, ZOOKEEPER_IMAGE +from .helpers import check_jaas_config, ping_servers logger = logging.getLogger(__name__) diff --git a/tests/integration/test_scaling.py b/tests/integration/test_scaling.py index 886478ad..8b57d5df 100644 --- a/tests/integration/test_scaling.py +++ b/tests/integration/test_scaling.py @@ -11,8 +11,8 @@ from lightkube.resources.core_v1 import Pod from pytest_operator.plugin import OpsTest -from tests.integration import APP_NAME, SERIES, ZOOKEEPER_IMAGE -from tests.integration.helpers import check_key, get_password, ping_servers, write_key +from . import APP_NAME, SERIES, ZOOKEEPER_IMAGE +from .helpers import check_key, get_password, ping_servers, write_key logger = logging.getLogger(__name__) diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 87ec3623..7e974e98 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -9,8 +9,8 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration import APP_NAME, SERIES, TLS_OPERATOR_SERIES, ZOOKEEPER_IMAGE -from tests.integration.helpers import check_properties, ping_servers +from . import APP_NAME, SERIES, TLS_OPERATOR_SERIES, ZOOKEEPER_IMAGE +from .helpers import check_properties, ping_servers logger = logging.getLogger(__name__) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..db3bfe1a --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tox.ini b/tox.ini index 17c349ec..7c1de83e 100644 --- a/tox.ini +++ b/tox.ini @@ -2,28 +2,28 @@ # See LICENSE file for licensing details. [tox] -skipsdist=True +no_package = True skip_missing_interpreters = True -envlist = lint, unit +env_list = lint, unit [vars] application = zookeeper-k8s -src_path = {toxinidir}/src/ -tst_path = {toxinidir}/tests/ -lib_path = {toxinidir}/lib/charms/zookeeper -all_path = {[vars]src_path} {[vars]tst_path} +src_path = {tox_root}/src +tests_path = {tox_root}/tests +lib_path = {tox_root}/lib/charms/zookeeper +all_path = {[vars]src_path} {[vars]tests_path} [testenv] allowlist_externals = /bin/bash -setenv = - PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path} - PYTHONBREAKPOINT=ipdb.set_trace - PY_COLORS=1 -passenv = - PYTHONPATH - CHARM_BUILD_DIR - MODEL_SETTINGS +set_env = + PYTHONPATH = {tox_root}/lib:{[vars]src_path} + PYTHONBREAKPOINT=ipdb.set_trace + PY_COLORS=1 +pass_env = + PYTHONPATH + CHARM_BUILD_DIR + MODEL_SETTINGS [testenv:refresh] description = Short refresh script for charm-dev @@ -34,7 +34,7 @@ commands = /bin/bash -ec "charmcraft pack" /bin/bash -ec "juju deploy ./*.charm -n 3 --resource zookeeper-image=ubuntu/zookeeper:latest" -[testenv:fmt] +[testenv:format] description = Apply coding style standards to code deps = black @@ -58,10 +58,10 @@ deps = commands = # uncomment the following line if this charm owns a lib codespell {[vars]lib_path} - codespell {toxinidir} --skip {toxinidir}/.git --skip {toxinidir}/.tox \ - --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/venv \ - --skip {toxinidir}/.mypy_cache --skip {toxinidir}/icon.svg \ - --skip {toxinidir}/LICENSE + codespell {tox_root} --skip {tox_root}/.git --skip {tox_root}/.tox \ + --skip {tox_root}/build --skip {tox_root}/lib --skip {tox_root}/venv \ + --skip {tox_root}/.mypy_cache --skip {tox_root}/icon.svg \ + --skip {tox_root}/LICENSE # pflake8 wrapper supports config from pyproject.toml pflake8 {[vars]all_path} isort --check-only --diff {[vars]all_path} @@ -74,50 +74,70 @@ deps = coverage[toml] cryptography jsonschema - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = coverage run --source={[vars]src_path} \ - -m pytest --ignore={[vars]tst_path}integration -vv --tb native -s {posargs} + -m pytest -vv --tb native -s {posargs} {[vars]tests_path}/unit coverage report [testenv:integration] description = Run integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = pytest juju==2.9.11 lightkube==0.11.0 pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -vv --no-header --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} + pytest -vv --no-header --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ [testenv:integration-provider] description = Run integration tests for provider +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = kazoo pure-sasl tenacity {[testenv:integration]deps} commands = - pytest tests/integration/test_provider.py -vv --no-header --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} + pytest -vv --no-header --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_provider.py [testenv:integration-password-rotation] description = Run integration tests for password-rotation +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = {[testenv:integration-provider]deps} commands = - pytest tests/integration/test_password_rotation.py -vv --no-header --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} + pytest -vv --no-header --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_password_rotation.py [testenv:integration-scaling] description = Run integration tests for scaling +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = {[testenv:integration-provider]deps} commands = - pytest tests/integration/test_scaling.py -vv --no-header --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} + pytest -vv --no-header --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_scaling.py [testenv:integration-tls] description = Run integration tests for tls +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = {[testenv:integration-provider]deps} commands = - pytest tests/integration/test_tls.py -vv --no-header --tb native --ignore={[vars]tst_path}unit + pytest -vv --no-header --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_tls.py From c52185415e84651e612f7db684c08d8166b81e23 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 27 Feb 2023 16:55:01 +0000 Subject: [PATCH 2/3] Update ci.yaml --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5fdce683..c3c89661 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,7 +36,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 integration-test: strategy: From af5a6bfd46ffdeea5cd7ccd3deb39121d6d2fff5 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 1 Mar 2023 16:46:01 +0000 Subject: [PATCH 3/3] Cancel CI that is not the latest commit on a branch Add time limit to jobs Ported from https://github.com/canonical/mysql-operator/pull/134 --- .github/workflows/ci.yaml | 7 +++++++ .github/workflows/release.yaml | 2 ++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c3c89661..97e78263 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,6 +2,10 @@ # See LICENSE file for licensing details. name: Tests +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: pull_request: schedule: @@ -13,6 +17,7 @@ jobs: lint: name: Lint runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v3 @@ -25,6 +30,7 @@ jobs: unit-test: name: Unit tests runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v3 @@ -53,6 +59,7 @@ jobs: - unit-test - build runs-on: ubuntu-latest + timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 098c3e25..721b22ef 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,6 +9,7 @@ jobs: lib-check: name: Check libraries runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v3 @@ -31,6 +32,7 @@ jobs: - lib-check - ci-tests runs-on: ubuntu-latest + timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@v3