From 105d82db10c57ff375d6a43041e697cbeeb78bdd Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Thu, 11 May 2023 09:08:01 -0400 Subject: [PATCH 1/9] Add CI rerun functionality --- tools/ci_status.py | 155 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/tools/ci_status.py b/tools/ci_status.py index be3061b9..909ec9e2 100644 --- a/tools/ci_status.py +++ b/tools/ci_status.py @@ -16,8 +16,10 @@ from typing import Optional import argparse +import time from github.Repository import Repository from github.Workflow import Workflow +from github.WorkflowRun import WorkflowRun from github.GithubException import GithubException from library_functions import StrPath from iterate_libraries import ( @@ -56,6 +58,46 @@ def run_gh_rest_check( return workflow_runs[0].conclusion +def run_gh_rest_rerun( + lib_repo: Repository, + user: Optional[str] = None, + branch: Optional[str] = None, + workflow_filename: Optional[str] = "build.yml", + rerun_level: int = 0, +) -> bool: + """Uses ``PyGithub`` to rerun the CI status of a repository + + :param Repository lib_repo: The repo as a github.Repository.Repository object + :param str|None user: The user that triggered the run; if `None` is + provided, any user is acceptable + :param str|None branch: The branch name to specifically check; if `None` is + provided, all branches are allowed; this is the default + :param str|None workflow_filename: The filename of the workflow; if `None` is + provided, any workflow name is acceptable; the default is ``"build.yml"`` + :param int rerun_level: The level at which rerun should occur (0 = none, + 1 = failed, 2 = all) + :return: The requested runs conclusion + :rtype: bool + """ + if not rerun_level: + return False + if rerun_level == 1: + result = ( + run_gh_rest_check(lib_repo, user, branch, workflow_filename) == "success" + ) + if rerun_level == 2 or not result: + arg_dict = {} + if user is not None: + arg_dict["actor"] = user + if branch is not None: + arg_dict["branch"] = branch + workflow: Workflow = lib_repo.get_workflow_run(workflow_filename) + latest_run: WorkflowRun = workflow.get_runs(**arg_dict)[0] + latest_run.rerun() + return True + return False + + def check_build_status( lib_repo: Repository, user: Optional[str] = None, @@ -105,6 +147,54 @@ def check_build_status( return None +def rerun_workflow( + lib_repo: Repository, + user: Optional[str] = None, + branch: Optional[str] = None, + workflow_filename: Optional[str] = "build.yml", + rerun_level: int = 0, + debug: bool = False, +): + """Uses ``PyGithub`` to rerun the CI of the Adafruit + CircuitPython Bundle repositories + + :param Repository lib_repo: The repo as a github.Repository.Repository object + :param str|None user: The user that triggered the run; if `None` is + provided, any user is acceptable + :param str|None branch: The branch name to specifically check; if `None` is + provided, all branches are allowed; this is the default + :param str|None workflow_filename: The filename of the workflow; if `None` + is provided, any workflow name is acceptable; the defail is `"build.yml"` + :param int rerun_level: The level at which rerun should occur (0 = none, + 1 = failed, 2 = all) + :param bool debug: Whether debug statements should be printed to the standard + output + :return: The result of the workflow run, or ``None`` if it could not be + determined + :rtype: bool|None + """ + if lib_repo.archived: + return False + + try: + result = run_gh_rest_rerun( + lib_repo, user, branch, workflow_filename, rerun_level + ) + if debug and result: + print("***", "Library", lib_repo.name, "workflow was rerun!", "***") + return result + except GithubException: + if debug: + print( + "???", + "Library", + lib_repo.name, + "had an issue occur", + "???", + ) + return None + + def check_build_statuses( gh_token: str, user: Optional[str] = None, @@ -136,6 +226,45 @@ def check_build_statuses( ) +def rerun_workflows( + gh_token: str, + user: Optional[str] = None, + branch: Optional[str] = "main", + workflow_filename: Optional[str] = "build.yml", + rerun_level: int = 0, + *, + debug: bool = False, +) -> list[RemoteLibFunc_IterResult[bool]]: + """Reruns the CI of all the libraries in the Adafruit CircuitPython Bundle. + + :param str gh_token: The Github token to be used for with the Github API + :param str|None user: The user that triggered the run; if `None` is + provided, any user is acceptable + :param str|None branch: The branch name to specifically check; if `None` is + provided, all branches are allowed; this is the default + :param str|None workflow_filename: The filename of the workflow; if `None` is + provided, any workflow name is acceptable; the defail is `"build.yml"` + :param int rerun_level: The level at which reruns should occur (0 = none, + 1 = failed, 2 = all) + :param bool debug: Whether debug statements should be printed to + the standard output + :return: A list of tuples containing paired Repoistory objects and build + statuses + :rtype: list + """ + + return iter_remote_bundle_with_func( + gh_token, + [ + ( + rerun_workflow, + (user, branch, workflow_filename, rerun_level), + {"debug": debug}, + ) + ], + ) + + def save_build_statuses( build_results: list[RemoteLibFunc_IterResult[bool]], failures_filepath: StrPath = "failures.txt", @@ -193,12 +322,38 @@ def save_build_statuses( parser.add_argument( "--debug", action="store_true", help="Print debug text during execution" ) + parser.add_argument( + "--rerun-level", + metavar="R", + type=int, + dest="rerun_level", + default=0, + help="Level to rerun CI workflows (0 = none, 1 = failed, 2 = all)", + ) args = parser.parse_args() + if args.rerun_level: + if args.debug: + print("Rerunning workflows...") + rerun_workflows( + args.gh_token, + args.user, + args.branch, + args.workflow, + args.rerun_level, + debug=args.debug, + ) + if args.debug: + print("Waiting 10 minutes to allow workflows to finish running...") + time.sleep(600) + + if args.debug: + print("Checking workflows statuses...") results = check_build_statuses( args.gh_token, args.user, args.branch, args.workflow, debug=args.debug ) + fail_list = [ repo_name.name for repo_name, repo_results in results if not repo_results[0] ] From 2aecdd7d1fa58f7d590a988e651ac5ecccd39a5e Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Wed, 10 May 2023 22:16:16 -0400 Subject: [PATCH 2/9] Add script for running pre-commit --- tools/run_black.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tools/run_black.sh diff --git a/tools/run_black.sh b/tools/run_black.sh new file mode 100644 index 00000000..e81438ba --- /dev/null +++ b/tools/run_black.sh @@ -0,0 +1,8 @@ +for repo in .libraries/*; do + cd $repo + pre-commit run --all-files + git add -A + git commit -m "Run pre-commit" + git push + cd .. +done From 1b814a01f39ef9b57dc26e06b238311e8bbf78a7 Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Thu, 11 May 2023 09:11:24 -0400 Subject: [PATCH 3/9] Update pre-commit script --- tools/run_black.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/run_black.sh b/tools/run_black.sh index e81438ba..5449b085 100644 --- a/tools/run_black.sh +++ b/tools/run_black.sh @@ -1,4 +1,9 @@ -for repo in .libraries/*; do +rm -rf .gitlibs +mkdir .gitlibs +cd .libraries +for repo in *; do + cd ../.gitlibs + git clone https://github.com/adafruit/$repo.git cd $repo pre-commit run --all-files git add -A From ba0716e895b7340e1c4d1a523e1451d82edab38d Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Thu, 11 May 2023 09:16:44 -0400 Subject: [PATCH 4/9] Add ability to use extra local folder of repos --- tools/ci_status.py | 17 ++++++++++-- tools/iterate_libraries.py | 54 ++++++++++++++++++++++++++++---------- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/tools/ci_status.py b/tools/ci_status.py index 909ec9e2..0cf5f0a9 100644 --- a/tools/ci_status.py +++ b/tools/ci_status.py @@ -91,7 +91,7 @@ def run_gh_rest_rerun( arg_dict["actor"] = user if branch is not None: arg_dict["branch"] = branch - workflow: Workflow = lib_repo.get_workflow_run(workflow_filename) + workflow: Workflow = lib_repo.get_workflow(workflow_filename) latest_run: WorkflowRun = workflow.get_runs(**arg_dict)[0] latest_run.rerun() return True @@ -202,6 +202,7 @@ def check_build_statuses( workflow_filename: Optional[str] = "build.yml", *, debug: bool = False, + local_folder: str = "", ) -> list[RemoteLibFunc_IterResult[bool]]: """Checks all the libraries in the Adafruit CircuitPython Bundle to get the latest build status with the requested information @@ -223,6 +224,7 @@ def check_build_statuses( return iter_remote_bundle_with_func( gh_token, [(check_build_status, (user, branch, workflow_filename), {"debug": debug})], + local_folder=local_folder, ) @@ -234,6 +236,7 @@ def rerun_workflows( rerun_level: int = 0, *, debug: bool = False, + local_folder: str = "", ) -> list[RemoteLibFunc_IterResult[bool]]: """Reruns the CI of all the libraries in the Adafruit CircuitPython Bundle. @@ -262,6 +265,7 @@ def rerun_workflows( {"debug": debug}, ) ], + local_folder=local_folder, ) @@ -330,6 +334,14 @@ def save_build_statuses( default=0, help="Level to rerun CI workflows (0 = none, 1 = failed, 2 = all)", ) + parser.add_argument( + "--local-folder", + metavar="L", + type=str, + dest="local_folder", + default="", + help="An additional folder to check and run" + ) args = parser.parse_args() @@ -343,6 +355,7 @@ def save_build_statuses( args.workflow, args.rerun_level, debug=args.debug, + local_folder=args.local_folder ) if args.debug: print("Waiting 10 minutes to allow workflows to finish running...") @@ -351,7 +364,7 @@ def save_build_statuses( if args.debug: print("Checking workflows statuses...") results = check_build_statuses( - args.gh_token, args.user, args.branch, args.workflow, debug=args.debug + args.gh_token, args.user, args.branch, args.workflow, debug=args.debug, local_folder=args.local_folder ) fail_list = [ diff --git a/tools/iterate_libraries.py b/tools/iterate_libraries.py index 6440b781..0e934fd3 100644 --- a/tools/iterate_libraries.py +++ b/tools/iterate_libraries.py @@ -16,8 +16,9 @@ import os import glob +import pathlib from collections.abc import Sequence, Iterable -from typing import TypeVar +from typing import TypeVar, Any, Callable, Union, List from typing_extensions import TypeAlias import parse from github import Github @@ -65,9 +66,20 @@ _BUNDLE_BRANCHES = ("drivers", "helpers") +def perform_func(item: Any, func_workflow: Union[RemoteLibFunc_IterInstruction, LocalLibFunc_IterInstruction]) -> Union[List[RemoteLibFunc_IterResult], List[LocalLibFunc_IterResult]]: + """ + Perform the given function + """ + func_results = [] + for func, args, kwargs in func_workflow: + result = func(item, *args, **kwargs) + func_results.append(result) + return func_results + + def iter_local_bundle_with_func( bundle_path: StrPath, - func_workflow: Iterable[LocalLibFunc_IterInstruction], + func_workflow: Iterable[LocalLibFunc_IterInstruction], *, local_folder: str = "", ) -> list[LocalLibFunc_IterResult]: """Iterate through the libraries and run a given function with the provided arguments @@ -85,6 +97,9 @@ def iter_local_bundle_with_func( # Initialize list of results results = [] + # Keep track of all libraries iterated + iterated = set() + # Loop through each bundle branch for branch_name in _BUNDLE_BRANCHES: @@ -94,20 +109,24 @@ def iter_local_bundle_with_func( # Enter each library in the bundle for library_path in libraries_path_list: - func_results = [] - - for func, args, kwargs in func_workflow: - result = func(library_path, *args, **kwargs) - func_results.append(result) + iterated.add(os.path.split(library_path).lower()) + func_results = perform_func(library_path, func_workflow) results.append((library_path, func_results)) + if local_folder: + additional = set(glob.glob(os.path.join(local_folder, "*"))) + diff = additional.difference(iterated) + for unused in diff: + unused_func_results = perform_func(unused, func_workflow) + results.append((unused, unused_func_results)) + return results # pylint: disable=too-many-locals def iter_remote_bundle_with_func( - gh_token: str, func_workflow: RemoteLibFunc_IterInstruction + gh_token: str, func_workflow: RemoteLibFunc_IterInstruction, *, local_folder: str = "", ) -> list[RemoteLibFunc_IterResult]: """Iterate through the remote bundle, accessing each library's git repo using the GitHub RESTful API (specifically using ``PyGithub``) @@ -129,6 +148,9 @@ def iter_remote_bundle_with_func( # Initialize list of results results = [] + # Keep track of all libraries iterated + iterated = set() + # Loop through each bundle branch for branch_name in _BUNDLE_BRANCHES: @@ -144,13 +166,17 @@ def iter_remote_bundle_with_func( repo_name: str = repo_name_result.named["repo_name"] repo = github_client.get_repo(f"adafruit/{repo_name}") + iterated.add(repo_name.lower()) - func_results = [] - - for func, args, kwargs in func_workflow: - result = func(repo, *args, **kwargs) - func_results.append(result) - + func_results = perform_func(repo, func_workflow) results.append((repo, func_results)) + if local_folder: + additional = {path.name.lower() for path in pathlib.Path(local_folder).glob("*")} + diff = additional.difference(iterated) + for unused in diff: + unused_repo = github_client.get_repo(f"adafruit/{unused}") + unused_func_results = perform_func(unused_repo, func_workflow) + results.append((unused_repo, unused_func_results)) + return results From 4f9c6fd5ef6b2c2d5a9513babcd754b7d8d8ab5e Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Thu, 11 May 2023 09:19:45 -0400 Subject: [PATCH 5/9] Add new temp folder to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6367b4b3..a1aed8ab 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ _build env.sh *.swp .libraries/* +.gitlibs/* .cp_org/* .blinka/* .vscode From 7b55088dfa50c5188b3286fbde1107536f1a4e8a Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Thu, 11 May 2023 09:31:59 -0400 Subject: [PATCH 6/9] Update docstrings for new arguments --- tools/ci_status.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ci_status.py b/tools/ci_status.py index 0cf5f0a9..e7cc0a77 100644 --- a/tools/ci_status.py +++ b/tools/ci_status.py @@ -216,6 +216,7 @@ def check_build_statuses( provided, any workflow name is acceptable; the defail is `"build.yml"` :param bool debug: Whether debug statements should be printed to the standard output + :param str local_folder: A path to a local folder containing extra repositories :return: A list of tuples containing paired Repoistory objects and build statuses :rtype: list @@ -251,6 +252,7 @@ def rerun_workflows( 1 = failed, 2 = all) :param bool debug: Whether debug statements should be printed to the standard output + :param str local_folder: A path to a local folder containing extra repositories :return: A list of tuples containing paired Repoistory objects and build statuses :rtype: list From 6b854cdd9d873f0341a87f95841c674aaac6dbc7 Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Thu, 11 May 2023 09:32:38 -0400 Subject: [PATCH 7/9] Reformatted per pre-commit --- tools/ci_status.py | 11 ++++++++--- tools/iterate_libraries.py | 18 ++++++++++++++---- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tools/ci_status.py b/tools/ci_status.py index e7cc0a77..b02315f1 100644 --- a/tools/ci_status.py +++ b/tools/ci_status.py @@ -342,7 +342,7 @@ def save_build_statuses( type=str, dest="local_folder", default="", - help="An additional folder to check and run" + help="An additional folder to check and run", ) args = parser.parse_args() @@ -357,7 +357,7 @@ def save_build_statuses( args.workflow, args.rerun_level, debug=args.debug, - local_folder=args.local_folder + local_folder=args.local_folder, ) if args.debug: print("Waiting 10 minutes to allow workflows to finish running...") @@ -366,7 +366,12 @@ def save_build_statuses( if args.debug: print("Checking workflows statuses...") results = check_build_statuses( - args.gh_token, args.user, args.branch, args.workflow, debug=args.debug, local_folder=args.local_folder + args.gh_token, + args.user, + args.branch, + args.workflow, + debug=args.debug, + local_folder=args.local_folder, ) fail_list = [ diff --git a/tools/iterate_libraries.py b/tools/iterate_libraries.py index 0e934fd3..e1e64123 100644 --- a/tools/iterate_libraries.py +++ b/tools/iterate_libraries.py @@ -66,7 +66,10 @@ _BUNDLE_BRANCHES = ("drivers", "helpers") -def perform_func(item: Any, func_workflow: Union[RemoteLibFunc_IterInstruction, LocalLibFunc_IterInstruction]) -> Union[List[RemoteLibFunc_IterResult], List[LocalLibFunc_IterResult]]: +def perform_func( + item: Any, + func_workflow: Union[RemoteLibFunc_IterInstruction, LocalLibFunc_IterInstruction], +) -> Union[List[RemoteLibFunc_IterResult], List[LocalLibFunc_IterResult]]: """ Perform the given function """ @@ -79,7 +82,9 @@ def perform_func(item: Any, func_workflow: Union[RemoteLibFunc_IterInstruction, def iter_local_bundle_with_func( bundle_path: StrPath, - func_workflow: Iterable[LocalLibFunc_IterInstruction], *, local_folder: str = "", + func_workflow: Iterable[LocalLibFunc_IterInstruction], + *, + local_folder: str = "", ) -> list[LocalLibFunc_IterResult]: """Iterate through the libraries and run a given function with the provided arguments @@ -126,7 +131,10 @@ def iter_local_bundle_with_func( # pylint: disable=too-many-locals def iter_remote_bundle_with_func( - gh_token: str, func_workflow: RemoteLibFunc_IterInstruction, *, local_folder: str = "", + gh_token: str, + func_workflow: RemoteLibFunc_IterInstruction, + *, + local_folder: str = "", ) -> list[RemoteLibFunc_IterResult]: """Iterate through the remote bundle, accessing each library's git repo using the GitHub RESTful API (specifically using ``PyGithub``) @@ -172,7 +180,9 @@ def iter_remote_bundle_with_func( results.append((repo, func_results)) if local_folder: - additional = {path.name.lower() for path in pathlib.Path(local_folder).glob("*")} + additional = { + path.name.lower() for path in pathlib.Path(local_folder).glob("*") + } diff = additional.difference(iterated) for unused in diff: unused_repo = github_client.get_repo(f"adafruit/{unused}") From 0087d98f40fa3e42a3884e2ecdabd06ed9b85175 Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Thu, 11 May 2023 09:33:24 -0400 Subject: [PATCH 8/9] Add REUSE header --- tools/run_black.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/run_black.sh b/tools/run_black.sh index 5449b085..a85bf7ad 100644 --- a/tools/run_black.sh +++ b/tools/run_black.sh @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2023 Alec Delaney +# +# SPDX-License-Identifier: MIT + rm -rf .gitlibs mkdir .gitlibs cd .libraries From 05b5de58c165dcbcb30df543342c3ae5ca66e18c Mon Sep 17 00:00:00 2001 From: Tekktrik Date: Thu, 11 May 2023 09:36:43 -0400 Subject: [PATCH 9/9] Linted per pre-commit --- tools/ci_status.py | 1 + tools/iterate_libraries.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/ci_status.py b/tools/ci_status.py index b02315f1..c7d830a9 100644 --- a/tools/ci_status.py +++ b/tools/ci_status.py @@ -147,6 +147,7 @@ def check_build_status( return None +# pylint: disable=too-many-arguments def rerun_workflow( lib_repo: Repository, user: Optional[str] = None, diff --git a/tools/iterate_libraries.py b/tools/iterate_libraries.py index e1e64123..d54996e7 100644 --- a/tools/iterate_libraries.py +++ b/tools/iterate_libraries.py @@ -18,7 +18,7 @@ import glob import pathlib from collections.abc import Sequence, Iterable -from typing import TypeVar, Any, Callable, Union, List +from typing import TypeVar, Any, Union, List from typing_extensions import TypeAlias import parse from github import Github @@ -114,13 +114,16 @@ def iter_local_bundle_with_func( # Enter each library in the bundle for library_path in libraries_path_list: - iterated.add(os.path.split(library_path).lower()) + iterated.add(os.path.split(library_path)[1].lower()) func_results = perform_func(library_path, func_workflow) results.append((library_path, func_results)) if local_folder: - additional = set(glob.glob(os.path.join(local_folder, "*"))) + additional = { + os.path.split(pathname)[1].lower() + for pathname in glob.glob(os.path.join(local_folder, "*")) + } diff = additional.difference(iterated) for unused in diff: unused_func_results = perform_func(unused, func_workflow)