From cb444d05b05f47572a17adfd79b4e50dea09dca2 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Mon, 26 Apr 2021 15:18:52 +0530 Subject: [PATCH 1/6] Add a util for configuring logging --- python/shared/log.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 python/shared/log.py diff --git a/python/shared/log.py b/python/shared/log.py new file mode 100644 index 00000000000..2b5b502057e --- /dev/null +++ b/python/shared/log.py @@ -0,0 +1,22 @@ +import logging +import os + + +def configure_logger(): + """ + Configures the logging module to + - change the log level names to lowercase + - set the formatter that works with GitHub logging commands + - set the default log level to LOGGING_LEVEL environment variable + """ + + logging.addLevelName(logging.CRITICAL, "critical") + logging.addLevelName(logging.ERROR, "error") + logging.addLevelName(logging.WARNING, "warning") + logging.addLevelName(logging.INFO, "info") + logging.addLevelName(logging.DEBUG, "debug") + + logging.basicConfig( + format="::%(levelname)s::[%(name)s] %(message)s", + level=int(os.getenv("LOGGING_LEVEL", logging.DEBUG)), + ) From 223b43f95176d7e5fdfbb5643fa1f3d6c4adc471 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Mon, 26 Apr 2021 15:19:28 +0530 Subject: [PATCH 2/6] Use the utility where logging is to be configured --- python/issues_with_prs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/issues_with_prs.py b/python/issues_with_prs.py index af436ed4935..67ebec7709c 100755 --- a/python/issues_with_prs.py +++ b/python/issues_with_prs.py @@ -1,6 +1,5 @@ import argparse import logging -import os from github import ( Github, @@ -14,8 +13,10 @@ from shared.data import get_data from shared.github import get_client +from shared.log import configure_logger + +configure_logger() -logging.basicConfig(level=int(os.getenv("LOGGING_LEVEL", logging.DEBUG))) log = logging.getLogger(__name__) # region argparse From c4d46fdd924a3b874bc4626615efcac1c35284d1 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 28 Apr 2021 18:02:54 +0530 Subject: [PATCH 3/6] Move project-related getters to shared module --- python/issues_with_prs.py | 41 +--------------------------------- python/shared/project.py | 47 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 40 deletions(-) create mode 100644 python/shared/project.py diff --git a/python/issues_with_prs.py b/python/issues_with_prs.py index 67ebec7709c..2e375ba1f9a 100755 --- a/python/issues_with_prs.py +++ b/python/issues_with_prs.py @@ -5,8 +5,6 @@ Github, GithubException, Issue, - Organization, - Project, ProjectColumn, ProjectCard, ) @@ -14,6 +12,7 @@ from shared.data import get_data from shared.github import get_client from shared.log import configure_logger +from shared.project import get_org_project, get_project_column configure_logger() @@ -85,44 +84,6 @@ def get_open_issues_with_prs( return all_issues -def get_org_project(org: Organization, proj_number: int) -> Project: - """ - Get the project with the given number in the given organization. - - :param org: the organization in which to find the project - :param proj_number: the number of the project to find in the organization - :return: the project being searched for - :raise: ValueError if no project found with given number - """ - - log.info(f"Getting project {proj_number} in org {org.name}") - projects = org.get_projects() - project = next(proj for proj in projects if proj.number == proj_number) - if project is None: - log.error(f"No project was found with number {proj_number}.") - raise ValueError(f"Project not found") - return project - - -def get_project_column(proj: Project, col_name: str) -> ProjectColumn: - """ - Get the project column with the given name in the given project. - - :param proj: the project in which to find the column - :param col_name: the name of the project column to find in the project - :return: the project column being searched for - :raise: ValueError if no project column found with given name - """ - - log.info(f"Getting column {col_name} in project {proj.name}") - columns = proj.get_columns() - column = next(col for col in columns if col.name == col_name) - if column is None: - log.error(f"No column was found with name {col_name}.") - raise ValueError(f"Column not found") - return column - - def get_issue_cards(col: ProjectColumn) -> list[ProjectCard]: """ Get all cards linked to issues in the given column. This excludes cards that diff --git a/python/shared/project.py b/python/shared/project.py new file mode 100644 index 00000000000..f2215bb1030 --- /dev/null +++ b/python/shared/project.py @@ -0,0 +1,47 @@ +import logging + +from github import ( + Organization, + Project, + ProjectColumn, +) + +log = logging.getLogger(__name__) + + +def get_org_project(org: Organization, proj_number: int) -> Project: + """ + Get the project with the given number in the given organization. + + :param org: the organization in which to find the project + :param proj_number: the number of the project to find in the organization + :return: the project being searched for + :raise: ValueError if no project found with given number + """ + + log.info(f"Getting project {proj_number} in org {org.name}") + projects = org.get_projects() + project = next(proj for proj in projects if proj.number == proj_number) + if project is None: + log.error(f"No project was found with number {proj_number}.") + raise ValueError(f"Project not found") + return project + + +def get_project_column(proj: Project, col_name: str) -> ProjectColumn: + """ + Get the project column with the given name in the given project. + + :param proj: the project in which to find the column + :param col_name: the name of the project column to find in the project + :return: the project column being searched for + :raise: ValueError if no project column found with given name + """ + + log.info(f"Getting column {col_name} in project {proj.name}") + columns = proj.get_columns() + column = next(col for col in columns if col.name == col_name) + if column is None: + log.error(f"No column was found with name {col_name}.") + raise ValueError(f"Column not found") + return column From 94effb7d23b10b329a11cddef971d122d741ac50 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 28 Apr 2021 18:03:34 +0530 Subject: [PATCH 4/6] Configure logger inside `__main__` flow --- python/issues_with_prs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/issues_with_prs.py b/python/issues_with_prs.py index 2e375ba1f9a..21346bbc3ff 100755 --- a/python/issues_with_prs.py +++ b/python/issues_with_prs.py @@ -14,8 +14,6 @@ from shared.log import configure_logger from shared.project import get_org_project, get_project_column -configure_logger() - log = logging.getLogger(__name__) # region argparse @@ -107,6 +105,8 @@ def get_issue_cards(col: ProjectColumn) -> list[ProjectCard]: if __name__ == "__main__": + configure_logger() + args = parser.parse_args() log.debug(f"Project number: {args.proj_number}") From dcf5d237761d89b84778de1b5815558799ead897 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 28 Apr 2021 18:04:14 +0530 Subject: [PATCH 5/6] Add code for adding new issues and PRs to project --- python/new_issues_and_prs.py | 155 +++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 python/new_issues_and_prs.py diff --git a/python/new_issues_and_prs.py b/python/new_issues_and_prs.py new file mode 100644 index 00000000000..d26bb033311 --- /dev/null +++ b/python/new_issues_and_prs.py @@ -0,0 +1,155 @@ +import argparse +import datetime +import logging +from collections import namedtuple + +from github import Github, Issue, GithubException +from github.PullRequest import PullRequest + +from shared.data import get_data +from shared.github import get_client +from shared.log import configure_logger +from shared.project import get_org_project, get_project_column + +log = logging.getLogger(__name__) + +EntityInfo = namedtuple("EntityInfo", ["display_name", "content_type"]) +ENTITY_INFO = { + "pr": EntityInfo("PR", "PullRequest"), + "issue": EntityInfo("issue", "Issue"), +} + +# region argparse +parser = argparse.ArgumentParser( + description="Move issues to the correct columns in projects", +) +parser.add_argument( + "--entity-type", + dest="entity_type", + metavar="entity-type", + type=str, + required=True, + choices=["issue", "pr"], + help="the type of entity to add to the project", +) +parser.add_argument( + "--project-number", + dest="proj_number", + metavar="project-number", + type=int, + required=True, + help="the project in which to add new cards with the entity", +) +parser.add_argument( + "--target-column", + dest="target_col_name", + metavar="target-column", + type=str, + default="Backlog", + help="column in which to add new cards with the entity", +) +parser.add_argument( + "--period", + type=int, # minutes + default=60, + help="time period in minutes within which to check for new issues", +) + + +# endregion + + +def get_new_issues( + gh: Github, + org_name: str, + repo_names: list[str], + ent_type: str, + since: datetime.datetime, +) -> list[Issue]: + """ + From given repos in the given organization, retrieve a list of open issues + that were created after the specified time. This includes PRs. + + :param gh: the GitHub client + :param org_name: the name of the org in which to look for issues + :param repo_names: the name of the repos in which to look for issues + :param ent_type: whether to retrieve issues or PRs (as issues) + :param since: the timestamp after which to retrieve + :return: the list of all retrieved entities + """ + + entity_info = ENTITY_INFO[ent_type] + all_entities = [] + for repo_name in repo_names: + log.info(f"Looking for {entity_info.display_name}s in {org_name}/{repo_name}") + entities = gh.search_issues( + query="", + sort="updated", + order="desc", + **{ + "repo": f"{org_name}/{repo_name}", + "is": ent_type, + "state": "open", + "created": f">={since.isoformat()}", + }, + ) + all_entities += list(entities) + + log.info(f"Found {len(all_entities)} new {entity_info.display_name}s created") + return all_entities + + +if __name__ == "__main__": + configure_logger() + + args = parser.parse_args() + + log.debug(f"Entity type: {args.entity_type}") + log.debug(f"Project number: {args.proj_number}") + log.debug(f"Target column name: {args.target_col_name}") + log.debug(f"Time period: {args.period}m") + + since = datetime.datetime.utcnow() - datetime.timedelta(minutes=args.period) + + github_info = get_data("github.yml") + org_name = github_info["org"] + log.info(f"Organization name: {org_name}") + repo_names = github_info["repos"].values() + log.info(f"Repository names: {', '.join(repo_names)}") + + gh = get_client() + org = gh.get_organization(org_name) + + entity_type = args.entity_type + entity_info = ENTITY_INFO[entity_type] + new_entities: list[Issue] = get_new_issues( + gh=gh, + org_name=org_name, + repo_names=repo_names, + ent_type=entity_type, + since=since, + ) + if entity_type == "pr": + new_entities: list[PullRequest] = [ + entity.as_pull_request() for entity in new_entities + ] + + proj = get_org_project(org=org, proj_number=args.proj_number) + log.info(f"Found project: {proj.name}") + target_column = get_project_column(proj=proj, col_name=args.target_col_name) + log.debug("Found target column") + + for entity in new_entities: + log.info(f"Creating card for {entity_info.display_name} {entity.number}") + try: + target_column.create_card( + content_id=entity.id, + content_type=entity_info.content_type, + ) + except GithubException as ex: + if "Project already has the associated" in str(ex): + log.warning(f"Card already exists") + else: + log.error( + f"Failed to create card for {entity_info.display_name} {entity.number}" + ) From 7cc2e09d33783c82c66edcbf8a6cf78d36c2e646 Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Wed, 28 Apr 2021 18:05:32 +0530 Subject: [PATCH 6/6] Use the new issues and PRs script in the automation workflow --- .github/workflows/project_automation.yml | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/project_automation.yml b/.github/workflows/project_automation.yml index be5a5955f10..41afc0a79f7 100644 --- a/.github/workflows/project_automation.yml +++ b/.github/workflows/project_automation.yml @@ -10,6 +10,62 @@ env: ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} jobs: + add_issues: + name: Add new issues + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install dependencies + working-directory: ./python + run: | + python -m pip install --user --upgrade pip + python -m pip install --user pipenv + pipenv install --deploy + + - name: Add issues to "Backlog" + working-directory: ./python + run: | + pipenv run python new_issues_and_prs.py \ + --entity-type "issue" \ + --project-number 242 \ + --target-column "Backlog" \ + --period 30 + + add_prs: + name: Add new PRs + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install dependencies + working-directory: ./python + run: | + python -m pip install --user --upgrade pip + python -m pip install --user pipenv + pipenv install --deploy + + - name: Add PRs to "In progress" + working-directory: ./python + run: | + pipenv run python new_issues_and_prs.py \ + --entity-type "pr" \ + --project-number 242 \ + --target-column "In progress" \ + --period 30 + move_issues: name: Move issues to "In Progress" runs-on: ubuntu-latest