From 29c5d70f1184cb56f74433e4fbf09cb47cde12c9 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 17 Feb 2023 21:23:15 +0000 Subject: [PATCH 1/6] Cache charmcraft pack container, skip unstable tests except on schedule Ported from https://github.com/canonical/mysql-k8s-operator/pull/146 --- .github/workflows/ci.yaml | 77 ++++++++++++++++++++------------- CONTRIBUTING.md | 16 +++---- pyproject.toml | 3 +- tests/integration/__init__.py | 2 + tests/integration/conftest.py | 25 ++++++++++- tests/integration/helpers.py | 2 +- tests/integration/test_charm.py | 4 +- tests/unit/__init__.py | 2 + tox.ini | 50 ++++++++++++--------- 9 files changed, 117 insertions(+), 64 deletions(-) create mode 100644 tests/integration/__init__.py create mode 100644 tests/unit/__init__.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 840a992..43bfc6f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,12 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. name: Tests + on: pull_request: + schedule: + - cron: '53 0 * * *' # Daily at 00:53 UTC + # Triggered on push to branch "main" by .github/workflows/release.yaml workflow_call: jobs: @@ -9,64 +15,75 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install tox run: python3 -m pip install tox - name: Install poetry run: curl -sSL https://install.python-poetry.org | python3 - - name: Run linters - run: tox -e lint + run: tox run -e lint + unit-test: name: Unit tests runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install tox run: python3 -m pip install tox - name: Install poetry run: curl -sSL https://install.python-poetry.org | python3 - - name: Run tests - run: tox -e unit - - integration-test-lxd: - name: Integration tests for (lxd) - runs-on: ubuntu-22.04 - needs: - - lint - - unit-test - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: lxd - bootstrap-options: "--agent-version 2.9.38" - - name: Install tox - run: python3 -m pip install tox - - name: Install poetry - run: curl -sSL https://install.python-poetry.org | python3 - - - name: Run integration tests - run: tox -e integration-lxd - - integration-test-k8s: - name: Integration tests for (microk8s) + run: tox run -e unit + + build: + name: Build charms + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1 + + integration-test: + strategy: + fail-fast: false + matrix: + tox-environments: + - integration-lxd: + provider: lxd + - integration-k8s: + provider: microk8s + name: ${{ matrix.tox-environments }} needs: - lint - unit-test + - build runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Setup operator environment + # TODO: Replace with custom image on self-hosted runner uses: charmed-kubernetes/actions-operator@main with: - provider: microk8s + provider: ${{ matrix.tox-environments.provider }} bootstrap-options: "--agent-version 2.9.38" - name: Install tox run: python3 -m pip install tox - name: Install poetry run: curl -sSL https://install.python-poetry.org | python3 - + - name: Download packed charm(s) + uses: actions/download-artifact@v3 + with: + name: ${{ needs.build.outputs.artifact-name }} + - name: Select tests + id: select-tests + run: | + if [ "${{ github.event_name }}" == "schedule" ] + then + echo Running unstable and stable tests + echo "mark_expression=" >> $GITHUB_OUTPUT + else + echo Skipping unstable tests + echo "mark_expression=not unstable" >> $GITHUB_OUTPUT + fi - name: Run integration tests - run: tox -e integration-k8s + run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' + env: + CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0404907..249ad3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,21 +47,21 @@ juju deploy ./*.charm ## Developing -You can use the environments created by `tox` for development: +You can create an environment for development with `tox`: ```shell -tox --notest -e unit -source .tox/unit/bin/activate +tox devenv -e integration +source venv/bin/activate ``` ### Testing ```shell -tox -e fmt # update your code according to linting rules -tox -e lint # code style -tox -e unit # unit tests -tox -e integration # integration tests -tox # runs 'lint' and 'unit' environments +tox run -e format # update your code according to linting rules +tox run -e lint # code style +tox run -e unit # unit tests +tox run -e integration # integration tests +tox # runs 'lint' and 'unit' environments ``` ## Canonical Contributor Agreement diff --git a/pyproject.toml b/pyproject.toml index b145031..23bdfcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,8 @@ show_missing = true [tool.pytest.ini_options] minversion = "6.0" log_cli_level = "INFO" -asyncio_mode = "auto" +asyncio_mode = "auto" +markers = ["unstable"] # Formatting tools configuration [tool.black] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..db3bfe1 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9ec9f1e..caf6f4d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,10 +1,33 @@ -# Copyright 2023 Canonical Ltd. +#!/usr/bin/env python3 +# Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. +import json +import os +from pathlib import Path + import pytest from pytest_operator.plugin import OpsTest +@pytest.fixture +def ops_test(ops_test: OpsTest) -> OpsTest: + if os.environ.get("CI") == "true": + # Running in GitHub Actions; skip build step + # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml) + packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"]) + + async def build_charm(charm_path, bases_index: int = None) -> Path: + for charm in packed_charms: + if Path(charm_path) == Path(charm["directory_path"]): + if bases_index is None or bases_index == charm["bases_index"]: + return charm["file_path"] + raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}") + + ops_test.build_charm = build_charm + return ops_test + + @pytest.fixture(scope="module") async def app_charm(ops_test: OpsTest): """Build the application charm.""" diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index ad06ea8..9d2cc34 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -8,7 +8,7 @@ from juju.unit import Unit -from tests.integration.constants import DATABASE_NAME, POSTGRESQL +from .constants import DATABASE_NAME, POSTGRESQL logger = logging.getLogger(__name__) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 048219b..14fe0fe 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -11,7 +11,7 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration.constants import ( +from .constants import ( APP, DATA_INTEGRATOR, DATABASE_NAME, @@ -23,7 +23,7 @@ TOPIC_NAME, ZOOKEEPER, ) -from tests.integration.helpers import ( +from .helpers import ( check_logs, fetch_action_database, fetch_action_get_credentials, diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..db3bfe1 --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tox.ini b/tox.ini index bd6408c..5c10a78 100644 --- a/tox.ini +++ b/tox.ini @@ -2,27 +2,27 @@ # See LICENSE file for licensing details. [tox] -skipsdist=True +no_package = True skip_missing_interpreters = True -envlist = fmt, lint, unit +env_list = lint, unit [vars] -src_path = {toxinidir}/src/ -tst_path = {toxinidir}/tests/ -lib_path = {toxinidir}/lib/ -all_path = {[vars]src_path} {[vars]tst_path} {[vars]lib_path} +src_path = {tox_root}/src +tests_path = {tox_root}/tests +lib_path = {tox_root}/lib +all_path = {[vars]src_path} {[vars]tests_path} {[vars]lib_path} [testenv] -setenv = - PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path} - PYTHONBREAKPOINT=ipdb.set_trace - PY_COLORS=1 -passenv = - PYTHONPATH - CHARM_BUILD_DIR - MODEL_SETTINGS +set_env = + PYTHONPATH = {tox_root}/lib:{[vars]src_path} + PYTHONBREAKPOINT=ipdb.set_trace + PY_COLORS=1 +pass_env = + PYTHONPATH + CHARM_BUILD_DIR + MODEL_SETTINGS -[testenv:fmt] +[testenv:format] allowlist_externals = poetry description = Apply coding style standards to code commands = @@ -36,9 +36,9 @@ description = Check code against coding style standards commands = poetry install poetry run codespell {[vars]lib_path} - poetry run codespell {toxinidir} --skip {toxinidir}/.git --skip {toxinidir}/.tox \ - --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/venv \ - --skip {toxinidir}/.mypy_cache --skip {toxinidir}/icon.svg + poetry run codespell {tox_root} --skip {tox_root}/.git --skip {tox_root}/.tox \ + --skip {tox_root}/build --skip {tox_root}/lib --skip {tox_root}/venv \ + --skip {tox_root}/.mypy_cache --skip {tox_root}/icon.svg # pflake8 wrapper supports config from pyproject.toml poetry run pflake8 {[vars]all_path} poetry run isort --check-only --diff {[vars]all_path} @@ -51,23 +51,31 @@ commands = poetry install poetry export -f requirements.txt -o requirements.txt poetry run coverage run --source={[vars]src_path} \ - -m pytest --ignore={[vars]tst_path}integration -v --tb native -s {posargs} + -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit poetry run coverage report [testenv:integration-lxd] allowlist_externals = poetry description = Run integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS commands = poetry install --with dev poetry export -f requirements.txt -o requirements.txt - poetry run pytest -v --tb native {[vars]tst_path}integration/test_charm.py --log-cli-level=INFO -s --cloud=localhost {posargs} + poetry run pytest -v --tb native --log-cli-level=INFO -s --cloud=localhost {posargs} {[vars]tests_path}/integration/test_charm.py [testenv:integration-k8s] allowlist_externals = poetry description = Run integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS commands = poetry install --with dev poetry export -f requirements.txt -o requirements.txt - poetry run pytest -v --tb native {[vars]tst_path}integration/test_charm.py --log-cli-level=INFO -s --cloud=microk8s {posargs} + poetry run pytest -v --tb native --log-cli-level=INFO -s --cloud=microk8s {posargs} {[vars]tests_path}/integration/test_charm.py From ae2bfd063138797e2600081c305d534e7b99e470 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 17 Feb 2023 21:23:42 +0000 Subject: [PATCH 2/6] Change scope of ops_test fixture to "module" --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index caf6f4d..6886558 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -10,7 +10,7 @@ from pytest_operator.plugin import OpsTest -@pytest.fixture +@pytest.fixture(scope="module") def ops_test(ops_test: OpsTest) -> OpsTest: if os.environ.get("CI") == "true": # Running in GitHub Actions; skip build step From ed500336f7992cd5ba01d028088b7d81693787ad Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 27 Feb 2023 16:55:27 +0000 Subject: [PATCH 3/6] Update build_charms_with_cache.yaml to v2 --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 43bfc6f..b89ca47 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,7 +38,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v1 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 integration-test: strategy: From a0864fc0f48e05e2e7175164f8b5673408d76551 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 1 Mar 2023 16:36:07 +0000 Subject: [PATCH 4/6] Cancel CI that is not the latest commit on a branch Add time limit to jobs Ported from https://github.com/canonical/mysql-operator/pull/134 --- .github/workflows/ci.yaml | 7 +++++++ .github/workflows/release.yaml | 2 ++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b89ca47..d660153 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,6 +2,10 @@ # See LICENSE file for licensing details. name: Tests +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: pull_request: schedule: @@ -13,6 +17,7 @@ jobs: lint: name: Lint runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v3 @@ -26,6 +31,7 @@ jobs: unit-test: name: Unit tests runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v3 @@ -55,6 +61,7 @@ jobs: - unit-test - build runs-on: ubuntu-latest + timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c0fbe14..94421be 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,6 +9,7 @@ jobs: lib-check: name: Check libraries runs-on: ubuntu-latest + timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v3 @@ -33,6 +34,7 @@ jobs: - lib-check - ci-tests runs-on: ubuntu-latest + timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@v3 From 2a7fc2ade82caf2b4a8607fb279f78f5b3876027 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 1 Mar 2023 17:15:05 +0000 Subject: [PATCH 5/6] Fix matrix --- .github/workflows/ci.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d660153..b861689 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -51,11 +51,11 @@ jobs: fail-fast: false matrix: tox-environments: - - integration-lxd: - provider: lxd - - integration-k8s: - provider: microk8s - name: ${{ matrix.tox-environments }} + - name: integration-lxd + provider: lxd + - name: integration-k8s + provider: microk8s + name: ${{ matrix.tox-environments.name }} needs: - lint - unit-test @@ -91,6 +91,6 @@ jobs: echo "mark_expression=not unstable" >> $GITHUB_OUTPUT fi - name: Run integration tests - run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' + run: tox run -e ${{ matrix.tox-environments.name }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' env: CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} From 991ad3e4ea9f629277cf4c2e143db8e7682d27b3 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 9 Mar 2023 15:20:22 +0000 Subject: [PATCH 6/6] Update tox.ini --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 5c10a78..9e9f2a8 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,7 @@ pass_env = commands = poetry install --with dev poetry export -f requirements.txt -o requirements.txt - poetry run pytest -v --tb native --log-cli-level=INFO -s --cloud=localhost {posargs} {[vars]tests_path}/integration/test_charm.py + poetry run pytest -v --tb native --log-cli-level=INFO -s --cloud=localhost {posargs} {[vars]tests_path}/integration/ [testenv:integration-k8s] allowlist_externals = poetry @@ -76,6 +76,6 @@ pass_env = commands = poetry install --with dev poetry export -f requirements.txt -o requirements.txt - poetry run pytest -v --tb native --log-cli-level=INFO -s --cloud=microk8s {posargs} {[vars]tests_path}/integration/test_charm.py + poetry run pytest -v --tb native --log-cli-level=INFO -s --cloud=microk8s {posargs} {[vars]tests_path}/integration/