feat: charm relation interfaces tests running on a schedule (#153)

* chore: rename interface.yaml to charms.yaml according to pytest-interface-tester

Signed-off-by: Tiexin Guo <[email protected]>

* test: run charm relation interface test on a schedule

---------

Signed-off-by: Tiexin Guo <[email protected]>

.github/workflows/matrix-tests.yaml

@@ -1,6 +1,9 @@
 name: Matrix tests for interfaces
 
-on: workflow_dispatch
+on:
+  schedule:
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
 
 jobs:
   main:
@@ -18,5 +21,6 @@ jobs:
       - name: Install dependencies
         run: pip install tox
       - name: Run tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: tox -e run-interface-test-matrix
-

pyproject.toml

@@ -23,7 +23,8 @@ keywords = ["juju", "relation interfaces"]
 
 dependencies = [
     "pydantic>2",
-    "pytest-interface-tester>=1.0.2"
+    "pytest-interface-tester>=3.1.0",
+    "PyGithub>=2.3.0"
 ]
 
 readme = "README.md"

run_matrix.py

@@ -10,8 +10,9 @@
 import subprocess
 from collections import namedtuple
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, Iterable, Literal, Tuple
+from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Tuple
 
+from github import Github
 from interface_tester.collector import collect_tests
 
 if TYPE_CHECKING:
@@ -77,6 +78,8 @@ def _prepare_repo(
     charm_config: "_CharmTestConfig",
     interface: str,
     version: int,
+    repo: str,
+    branch: str,
     root: Path = Path("/tmp/charm-relation-interfaces-tests/"),
 ) -> Tuple[Path, Path]:
     """Clone the charm repository and create the venv if it hasn't been done already."""
@@ -94,7 +97,7 @@ def _prepare_repo(
         # that the charm needs no patching at all to work with scenario
         raise SetupError(f"fixture missing for charm {charm_config.name}")
     test_path = _generate_test(
-        interface, fixture_spec.path.parent, fixture_spec.id, version
+        interface, fixture_spec.path.parent, fixture_spec.id, version, repo, branch
     )
     return charm_path, test_path
 
@@ -112,18 +115,29 @@ def test_{interface}_interface({fixture_id}: InterfaceTester):
     {fixture_id}.configure(
         interface_name="{interface}",
         interface_version={version},
+        repo="{repo}",
+        branch="{branch}",
     )
     {fixture_id}.run()
 """
 
 
 def _generate_test(
-    interface: str, test_path: Path, fixture_id: str, version: int
+    interface: str,
+    test_path: Path,
+    fixture_id: str,
+    version: int,
+    repo: str,
+    branch: str,
 ) -> Path:
     """Generate a pytest file for a given charm and interface."""
     logging.info(f"Generating test file for {interface} at {test_path}")
     test_content = _TEST_CONTENT.format(
-        interface=interface, fixture_id=fixture_id, version=version
+        interface=interface,
+        fixture_id=fixture_id,
+        version=version,
+        repo=repo,
+        branch=branch,
     )
     test_filename = f"interface_test_{interface}.py"
     with open(test_path / test_filename, "w") as file:
@@ -191,12 +205,19 @@ def _run_test_with_pytest(root: Path, test_path: Path):
 
 
 def _test_charm(
-    charm_config: "_CharmTestConfig", interface: str, version: int, role: str
+    charm_config: "_CharmTestConfig",
+    interface: str,
+    version: int,
+    role: str,
+    repo: str,
+    branch: str,
 ) -> bool:
     """Run interface tests for a charm."""
     logging.info(f"Running tests for charm: {charm_config.name}")
     try:
-        charm_path, test_path = _prepare_repo(charm_config, interface, version)
+        charm_path, test_path = _prepare_repo(
+            charm_config, interface, version, repo, branch
+        )
     except SetupError:
         logging.warning(
             f"test setup failed for {charm_config.name} {interface} {role}",
@@ -216,20 +237,29 @@ def _test_charm(
 
 
 def _test_charms(
-    charm_configs: Iterable["_CharmTestConfig"], interface: str, version: int, role: str
+    charm_configs: Iterable["_CharmTestConfig"],
+    interface: str,
+    version: int,
+    role: str,
+    repo: str,
+    branch: str,
 ) -> "_ResultsPerCharm":
     """Test all charms against this interface and role."""
     logging.info(f"Running tests for {interface}")
     out = {}
     for charm_config in charm_configs:
-        success = _test_charm(charm_config, interface, version, role)
+        success = _test_charm(charm_config, interface, version, role, repo, branch)
         out[charm_config.name] = success
         logging.info(f"Result: {'PASSED' if success else 'FAILED'}")
     return out
 
 
 def _test_roles(
-    tests_per_role: Dict["_Role", "_RoleTestSpec"], interface: str, version: int
+    tests_per_role: Dict["_Role", "_RoleTestSpec"],
+    interface: str,
+    version: int,
+    repo: str,
+    branch: str,
 ) -> "_ResultsPerRole":
     """Run the tests for each role of this interface."""
     results_per_role: _ResultsPerRole = {}
@@ -251,69 +281,167 @@ def _test_roles(
                 f"{[charm.name for charm in charm_configs]}..."
             )
             results_per_role[role] = _test_charms(
-                charm_configs, interface, version, role
+                charm_configs, interface, version, role, repo, branch
             )
     return results_per_role
 
 
-def _test_interface_version(tests_per_version, interface: str) -> "_ResultsPerVersion":
+def _test_interface_version(
+    tests_per_version, interface: str, repo: str, branch: str
+) -> "_ResultsPerVersion":
     """Run the tests for each version of this interface."""
     logging.info(f"Running tests for interface: {interface}")
     results_per_version: _ResultsPerVersion = {}
-
     for version, tests_per_role in tests_per_version.items():
         logging.info(f"Running tests for version: {version}")
 
         version_int = int(version[1:])
         results_per_version[version] = _test_roles(
-            tests_per_role, interface, version_int
+            tests_per_role, interface, version_int, repo, branch
         )
 
     return results_per_version
 
 
 def run_interface_tests(
-    path: Path, include: str = "*", keep_cache: bool = False
+    path: Path,
+    repo: str,
+    branch: str,
+    include: str = "*",
+    keep_cache: bool = False,
 ) -> "_ResultsPerInterface":
     """Run the tests for the specified interfaces, defaulting to all."""
     if not keep_cache:
         _clean()
     test_results = {}
     collected = collect_tests(path=path, include=include)
     for interface, version_to_roles in collected.items():
-        results_per_version = _test_interface_version(version_to_roles, interface)
+        results_per_version = _test_interface_version(
+            version_to_roles, interface, repo, branch
+        )
         test_results[interface] = results_per_version
 
+        # Running in GitHub actions with owners set on the test.
+        if os.getenv("GITHUB_ACTIONS"):
+            for version, tests_per_role in version_to_roles.items():
+                owners = tests_per_role.get("owners")
+                if owners and test_failed(results_per_version[version]):
+                    create_issue(
+                        interface, version, results_per_version[version], owners
+                    )
+
     if not collected:
         logging.warning("No tests collected.")
 
     return test_results
 
 
+def test_failed(role_result: "_ResultsPerRole"):
+    for charm_result in role_result.values():
+        if False in charm_result.values():
+            return True
+
+    return False
+
+
+def create_issue(
+    interface: str, version: str, result_per_role: "_ResultsPerRole", owners: List[str]
+):
+    gh = Github(os.getenv("GITHUB_TOKEN"))
+    repo = gh.get_repo("canonical/charm-relation-interfaces")
+    workflow_url = ""
+    github_run_id = os.getenv("GITHUB_RUN_ID")
+    if github_run_id:
+        workflow_url = f"https://github.com/canonical/charm-relation-interfaces/actions/runs/{github_run_id}"
+    result = flatten_test_result(result_per_role)
+    title = f"Interface test for {interface} {version} failed."
+    body = f"""\
+Tests for interface {interface} {version} failed.
+
+{result}
+
+See the workflow {workflow_url} for more detail.
+"""
+
+    issue = None
+    labels = ["Type: Interface Testing"]
+    for existing_issue in repo.get_issues(state="open", labels=labels):
+        if f"{interface} {version}" in existing_issue.title:
+            issue = existing_issue
+            break
+
+    if issue:
+        issue.create_comment(body)
+        print(f"GitHub issue updated: {issue.html_url}")
+        if owners:
+            issue.edit(assignees=owners)
+            print(f"GitHub issue assigned to {owners}")
+    else:
+        issue = repo.create_issue(
+            title=title, body=body, assignees=owners, labels=labels
+        )
+        print(f"GitHub issue created: {issue.html_url}")
+
+
+def flatten_test_result(version_result: "_ResultsPerRole"):
+    result = ""
+
+    provider_res = "\n".join(
+        f"- {charm}: {res}"
+        for charm, res in version_result.get("provider").items()
+        if res is False
+    )
+
+    if provider_res:
+        result = "## Provider\n\n" + provider_res
+
+    requirer_res = "\n".join(
+        f"- {charm}: {res}"
+        for charm, res in version_result.get("requirer").items()
+        if res is False
+    )
+
+    if requirer_res:
+        result += "## Requirer\n\n" + requirer_res
+
+    return result
+
+
 def pprint_interface_test_results(test_results: dict):
     """Pretty print the results of interface tests."""
     print("+++ Results +++")
     print(json.dumps(test_results, indent=2))
 
 
 if __name__ == "__main__":
-    # import argparse
-    #
-    # parser = argparse.ArgumentParser()
-    # parser.add_argument(
-    #     "--include",
-    #     default="*",
-    #     help="Glob to filter what interfaces to include in the test matrix.",
-    # )
-    # parser.add_argument(
-    #     "--keep-cache",
-    #     default=False,
-    #     help="Keep the charm cache intact before running the tests. "
-    #          "This will save some time when running the tests again "
-    #          "(assuming the charms haven't changed).",
-    # )
-    # args = parser.parse_args()
-
-    # result = run_interface_tests(Path("."), args.include, args.keep_cache)
-    result = run_interface_tests(Path("."), "tracing", False)
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--include",
+        default="*",
+        help="Glob to filter what interfaces to include in the test matrix.",
+    )
+    parser.add_argument(
+        "--keep-cache",
+        default=False,
+        help="Keep the charm cache intact before running the tests. "
+        "This will save some time when running the tests again "
+        "(assuming the charms haven't changed).",
+    )
+    parser.add_argument(
+        "--repo",
+        default="https://github.com/canonical/charm-relation-interfaces",
+        help="The repository where to find the tests, defaults to https://github.com/canonical/charm-relation-interfaces.",
+    )
+    parser.add_argument(
+        "--branch",
+        default="main",
+        help="The branch of the repo where to find the tests, defaults to main.",
+    )
+    args = parser.parse_args()
+
+    result = run_interface_tests(
+        Path("."), args.repo, args.branch, args.include, args.keep_cache
+    )
     pprint_interface_test_results(result)

tox.ini

@@ -79,6 +79,10 @@ deps =
     .[interface_tests]
 setenv =
     PYTHONPATH={toxinidir}
+passenv =
+    GITHUB_TOKEN
+    GITHUB_RUN_ID
+    GITHUB_ACTIONS
 commands =
     python {toxinidir}/run_matrix.py {posargs}
 

utils/interface-validator.py

@@ -44,6 +44,7 @@ class InterfaceModel(BaseModel):
     status: StatusEnum
     requirers: List[CharmEntry]
     providers: List[CharmEntry]
+    owners: Optional[List[str]] = []
 
 class MatchError(Exception):
     """Error raised when the location of an interface.yaml spec file is inconsistent with its contents."""