From 784ea23da89007b18df6ccc1353a5cd02b68f2f5 Mon Sep 17 00:00:00 2001 From: Tiexin Guo Date: Tue, 13 Aug 2024 16:12:18 +0800 Subject: [PATCH] feat: charm relation interfaces tests running on a schedule (#153) * chore: rename interface.yaml to charms.yaml according to pytest-interface-tester Signed-off-by: Tiexin Guo * test: run charm relation interface test on a schedule --------- Signed-off-by: Tiexin Guo --- .github/workflows/matrix-tests.yaml | 8 +- pyproject.toml | 3 +- run_matrix.py | 196 +++++++++++++++++++++++----- tox.ini | 4 + utils/interface-validator.py | 1 + 5 files changed, 175 insertions(+), 37 deletions(-) diff --git a/.github/workflows/matrix-tests.yaml b/.github/workflows/matrix-tests.yaml index f37aeb2f..084663f7 100644 --- a/.github/workflows/matrix-tests.yaml +++ b/.github/workflows/matrix-tests.yaml @@ -1,6 +1,9 @@ name: Matrix tests for interfaces -on: workflow_dispatch +on: + schedule: + - cron: '0 0 * * 1' + workflow_dispatch: jobs: main: @@ -18,5 +21,6 @@ jobs: - name: Install dependencies run: pip install tox - name: Run tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: tox -e run-interface-test-matrix - diff --git a/pyproject.toml b/pyproject.toml index a4a24727..4a1ef801 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,8 @@ keywords = ["juju", "relation interfaces"] dependencies = [ "pydantic>2", - "pytest-interface-tester>=1.0.2" + "pytest-interface-tester>=3.1.0", + "PyGithub>=2.3.0" ] readme = "README.md" diff --git a/run_matrix.py b/run_matrix.py index 065b93e1..acf3079d 100644 --- a/run_matrix.py +++ b/run_matrix.py @@ -10,8 +10,9 @@ import subprocess from collections import namedtuple from pathlib import Path -from typing import TYPE_CHECKING, Dict, Iterable, Literal, Tuple +from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Tuple +from github import Github from interface_tester.collector import collect_tests if TYPE_CHECKING: @@ -77,6 +78,8 @@ def _prepare_repo( charm_config: "_CharmTestConfig", interface: str, version: int, + repo: str, + branch: str, root: Path = Path("/tmp/charm-relation-interfaces-tests/"), ) -> Tuple[Path, Path]: """Clone the charm repository and create the venv if it hasn't been done already.""" @@ -94,7 +97,7 @@ def _prepare_repo( # that the charm needs no patching at all to work with scenario raise SetupError(f"fixture missing for charm {charm_config.name}") test_path = _generate_test( - interface, fixture_spec.path.parent, fixture_spec.id, version + interface, fixture_spec.path.parent, fixture_spec.id, version, repo, branch ) return charm_path, test_path @@ -112,18 +115,29 @@ def test_{interface}_interface({fixture_id}: InterfaceTester): {fixture_id}.configure( interface_name="{interface}", interface_version={version}, + repo="{repo}", + branch="{branch}", ) {fixture_id}.run() """ def _generate_test( - interface: str, test_path: Path, fixture_id: str, version: int + interface: str, + test_path: Path, + fixture_id: str, + version: int, + repo: str, + branch: str, ) -> Path: """Generate a pytest file for a given charm and interface.""" logging.info(f"Generating test file for {interface} at {test_path}") test_content = _TEST_CONTENT.format( - interface=interface, fixture_id=fixture_id, version=version + interface=interface, + fixture_id=fixture_id, + version=version, + repo=repo, + branch=branch, ) test_filename = f"interface_test_{interface}.py" with open(test_path / test_filename, "w") as file: @@ -191,12 +205,19 @@ def _run_test_with_pytest(root: Path, test_path: Path): def _test_charm( - charm_config: "_CharmTestConfig", interface: str, version: int, role: str + charm_config: "_CharmTestConfig", + interface: str, + version: int, + role: str, + repo: str, + branch: str, ) -> bool: """Run interface tests for a charm.""" logging.info(f"Running tests for charm: {charm_config.name}") try: - charm_path, test_path = _prepare_repo(charm_config, interface, version) + charm_path, test_path = _prepare_repo( + charm_config, interface, version, repo, branch + ) except SetupError: logging.warning( f"test setup failed for {charm_config.name} {interface} {role}", @@ -216,20 +237,29 @@ def _test_charm( def _test_charms( - charm_configs: Iterable["_CharmTestConfig"], interface: str, version: int, role: str + charm_configs: Iterable["_CharmTestConfig"], + interface: str, + version: int, + role: str, + repo: str, + branch: str, ) -> "_ResultsPerCharm": """Test all charms against this interface and role.""" logging.info(f"Running tests for {interface}") out = {} for charm_config in charm_configs: - success = _test_charm(charm_config, interface, version, role) + success = _test_charm(charm_config, interface, version, role, repo, branch) out[charm_config.name] = success logging.info(f"Result: {'PASSED' if success else 'FAILED'}") return out def _test_roles( - tests_per_role: Dict["_Role", "_RoleTestSpec"], interface: str, version: int + tests_per_role: Dict["_Role", "_RoleTestSpec"], + interface: str, + version: int, + repo: str, + branch: str, ) -> "_ResultsPerRole": """Run the tests for each role of this interface.""" results_per_role: _ResultsPerRole = {} @@ -251,29 +281,34 @@ def _test_roles( f"{[charm.name for charm in charm_configs]}..." ) results_per_role[role] = _test_charms( - charm_configs, interface, version, role + charm_configs, interface, version, role, repo, branch ) return results_per_role -def _test_interface_version(tests_per_version, interface: str) -> "_ResultsPerVersion": +def _test_interface_version( + tests_per_version, interface: str, repo: str, branch: str +) -> "_ResultsPerVersion": """Run the tests for each version of this interface.""" logging.info(f"Running tests for interface: {interface}") results_per_version: _ResultsPerVersion = {} - for version, tests_per_role in tests_per_version.items(): logging.info(f"Running tests for version: {version}") version_int = int(version[1:]) results_per_version[version] = _test_roles( - tests_per_role, interface, version_int + tests_per_role, interface, version_int, repo, branch ) return results_per_version def run_interface_tests( - path: Path, include: str = "*", keep_cache: bool = False + path: Path, + repo: str, + branch: str, + include: str = "*", + keep_cache: bool = False, ) -> "_ResultsPerInterface": """Run the tests for the specified interfaces, defaulting to all.""" if not keep_cache: @@ -281,15 +316,97 @@ def run_interface_tests( test_results = {} collected = collect_tests(path=path, include=include) for interface, version_to_roles in collected.items(): - results_per_version = _test_interface_version(version_to_roles, interface) + results_per_version = _test_interface_version( + version_to_roles, interface, repo, branch + ) test_results[interface] = results_per_version + # Running in GitHub actions with owners set on the test. + if os.getenv("GITHUB_ACTIONS"): + for version, tests_per_role in version_to_roles.items(): + owners = tests_per_role.get("owners") + if owners and test_failed(results_per_version[version]): + create_issue( + interface, version, results_per_version[version], owners + ) + if not collected: logging.warning("No tests collected.") return test_results +def test_failed(role_result: "_ResultsPerRole"): + for charm_result in role_result.values(): + if False in charm_result.values(): + return True + + return False + + +def create_issue( + interface: str, version: str, result_per_role: "_ResultsPerRole", owners: List[str] +): + gh = Github(os.getenv("GITHUB_TOKEN")) + repo = gh.get_repo("canonical/charm-relation-interfaces") + workflow_url = "" + github_run_id = os.getenv("GITHUB_RUN_ID") + if github_run_id: + workflow_url = f"https://github.com/canonical/charm-relation-interfaces/actions/runs/{github_run_id}" + result = flatten_test_result(result_per_role) + title = f"Interface test for {interface} {version} failed." + body = f"""\ +Tests for interface {interface} {version} failed. + +{result} + +See the workflow {workflow_url} for more detail. +""" + + issue = None + labels = ["Type: Interface Testing"] + for existing_issue in repo.get_issues(state="open", labels=labels): + if f"{interface} {version}" in existing_issue.title: + issue = existing_issue + break + + if issue: + issue.create_comment(body) + print(f"GitHub issue updated: {issue.html_url}") + if owners: + issue.edit(assignees=owners) + print(f"GitHub issue assigned to {owners}") + else: + issue = repo.create_issue( + title=title, body=body, assignees=owners, labels=labels + ) + print(f"GitHub issue created: {issue.html_url}") + + +def flatten_test_result(version_result: "_ResultsPerRole"): + result = "" + + provider_res = "\n".join( + f"- {charm}: {res}" + for charm, res in version_result.get("provider").items() + if res is False + ) + + if provider_res: + result = "## Provider\n\n" + provider_res + + requirer_res = "\n".join( + f"- {charm}: {res}" + for charm, res in version_result.get("requirer").items() + if res is False + ) + + if requirer_res: + result += "## Requirer\n\n" + requirer_res + + return result + + def pprint_interface_test_results(test_results: dict): """Pretty print the results of interface tests.""" print("+++ Results +++") @@ -297,23 +414,34 @@ def pprint_interface_test_results(test_results: dict): if __name__ == "__main__": - # import argparse - # - # parser = argparse.ArgumentParser() - # parser.add_argument( - # "--include", - # default="*", - # help="Glob to filter what interfaces to include in the test matrix.", - # ) - # parser.add_argument( - # "--keep-cache", - # default=False, - # help="Keep the charm cache intact before running the tests. " - # "This will save some time when running the tests again " - # "(assuming the charms haven't changed).", - # ) - # args = parser.parse_args() - - # result = run_interface_tests(Path("."), args.include, args.keep_cache) - result = run_interface_tests(Path("."), "tracing", False) + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--include", + default="*", + help="Glob to filter what interfaces to include in the test matrix.", + ) + parser.add_argument( + "--keep-cache", + default=False, + help="Keep the charm cache intact before running the tests. " + "This will save some time when running the tests again " + "(assuming the charms haven't changed).", + ) + parser.add_argument( + "--repo", + default="https://github.com/canonical/charm-relation-interfaces", + help="The repository where to find the tests, defaults to https://github.com/canonical/charm-relation-interfaces.", + ) + parser.add_argument( + "--branch", + default="main", + help="The branch of the repo where to find the tests, defaults to main.", + ) + args = parser.parse_args() + + result = run_interface_tests( + Path("."), args.repo, args.branch, args.include, args.keep_cache + ) pprint_interface_test_results(result) diff --git a/tox.ini b/tox.ini index 47c465b6..7e620894 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,10 @@ deps = .[interface_tests] setenv = PYTHONPATH={toxinidir} +passenv = + GITHUB_TOKEN + GITHUB_RUN_ID + GITHUB_ACTIONS commands = python {toxinidir}/run_matrix.py {posargs} diff --git a/utils/interface-validator.py b/utils/interface-validator.py index 15488eb5..f3d4f9ed 100644 --- a/utils/interface-validator.py +++ b/utils/interface-validator.py @@ -44,6 +44,7 @@ class InterfaceModel(BaseModel): status: StatusEnum requirers: List[CharmEntry] providers: List[CharmEntry] + owners: Optional[List[str]] = [] class MatchError(Exception): """Error raised when the location of an interface.yaml spec file is inconsistent with its contents."""