Improve the logging format

.github/workflows/project_automation.yml

@@ -10,6 +10,62 @@ env:
   ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
 
 jobs:
+  add_issues:
+    name: Add new issues
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      working-directory: ./python
+      run: |
+        python -m pip install --user --upgrade pip
+        python -m pip install --user pipenv
+        pipenv install --deploy
+
+    - name: Add issues to "Backlog"
+      working-directory: ./python
+      run: |
+        pipenv run python new_issues_and_prs.py \
+        --entity-type "issue" \
+        --project-number 242 \
+        --target-column "Backlog" \
+        --period 30
+
+  add_prs:
+    name: Add new PRs
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      working-directory: ./python
+      run: |
+        python -m pip install --user --upgrade pip
+        python -m pip install --user pipenv
+        pipenv install --deploy
+
+    - name: Add PRs to "In progress"
+      working-directory: ./python
+      run: |
+        pipenv run python new_issues_and_prs.py \
+        --entity-type "pr" \
+        --project-number 242 \
+        --target-column "In progress" \
+        --period 30
+
   move_issues:
     name: Move issues to "In Progress"
     runs-on: ubuntu-latest

python/issues_with_prs.py

@@ -1,21 +1,19 @@
 import argparse
 import logging
-import os
 
 from github import (
     Github,
     GithubException,
     Issue,
-    Organization,
-    Project,
     ProjectColumn,
     ProjectCard,
 )
 
 from shared.data import get_data
 from shared.github import get_client
+from shared.log import configure_logger
+from shared.project import get_org_project, get_project_column
 
-logging.basicConfig(level=int(os.getenv("LOGGING_LEVEL", logging.DEBUG)))
 log = logging.getLogger(__name__)
 
 # region argparse
@@ -84,44 +82,6 @@ def get_open_issues_with_prs(
     return all_issues
 
 
-def get_org_project(org: Organization, proj_number: int) -> Project:
-    """
-    Get the project with the given number in the given organization.
-
-    :param org: the organization in which to find the project
-    :param proj_number: the number of the project to find in the organization
-    :return: the project being searched for
-    :raise: ValueError if no project found with given number
-    """
-
-    log.info(f"Getting project {proj_number} in org {org.name}")
-    projects = org.get_projects()
-    project = next(proj for proj in projects if proj.number == proj_number)
-    if project is None:
-        log.error(f"No project was found with number {proj_number}.")
-        raise ValueError(f"Project not found")
-    return project
-
-
-def get_project_column(proj: Project, col_name: str) -> ProjectColumn:
-    """
-    Get the project column with the given name in the given project.
-
-    :param proj: the project in which to find the column
-    :param col_name: the name of the project column to find in the project
-    :return: the project column being searched for
-    :raise: ValueError if no project column found with given name
-    """
-
-    log.info(f"Getting column {col_name} in project {proj.name}")
-    columns = proj.get_columns()
-    column = next(col for col in columns if col.name == col_name)
-    if column is None:
-        log.error(f"No column was found with name {col_name}.")
-        raise ValueError(f"Column not found")
-    return column
-
-
 def get_issue_cards(col: ProjectColumn) -> list[ProjectCard]:
     """
     Get all cards linked to issues in the given column. This excludes cards that
@@ -145,6 +105,8 @@ def get_issue_cards(col: ProjectColumn) -> list[ProjectCard]:
 
 
 if __name__ == "__main__":
+    configure_logger()
+
     args = parser.parse_args()
 
     log.debug(f"Project number: {args.proj_number}")

python/new_issues_and_prs.py

@@ -0,0 +1,155 @@
+import argparse
+import datetime
+import logging
+from collections import namedtuple
+
+from github import Github, Issue, GithubException
+from github.PullRequest import PullRequest
+
+from shared.data import get_data
+from shared.github import get_client
+from shared.log import configure_logger
+from shared.project import get_org_project, get_project_column
+
+log = logging.getLogger(__name__)
+
+EntityInfo = namedtuple("EntityInfo", ["display_name", "content_type"])
+ENTITY_INFO = {
+    "pr": EntityInfo("PR", "PullRequest"),
+    "issue": EntityInfo("issue", "Issue"),
+}
+
+# region argparse
+parser = argparse.ArgumentParser(
+    description="Move issues to the correct columns in projects",
+)
+parser.add_argument(
+    "--entity-type",
+    dest="entity_type",
+    metavar="entity-type",
+    type=str,
+    required=True,
+    choices=["issue", "pr"],
+    help="the type of entity to add to the project",
+)
+parser.add_argument(
+    "--project-number",
+    dest="proj_number",
+    metavar="project-number",
+    type=int,
+    required=True,
+    help="the project in which to add new cards with the entity",
+)
+parser.add_argument(
+    "--target-column",
+    dest="target_col_name",
+    metavar="target-column",
+    type=str,
+    default="Backlog",
+    help="column in which to add new cards with the entity",
+)
+parser.add_argument(
+    "--period",
+    type=int,  # minutes
+    default=60,
+    help="time period in minutes within which to check for new issues",
+)
+
+
+# endregion
+
+
+def get_new_issues(
+    gh: Github,
+    org_name: str,
+    repo_names: list[str],
+    ent_type: str,
+    since: datetime.datetime,
+) -> list[Issue]:
+    """
+    From given repos in the given organization, retrieve a list of open issues
+    that were created after the specified time. This includes PRs.
+
+    :param gh: the GitHub client
+    :param org_name: the name of the org in which to look for issues
+    :param repo_names: the name of the repos in which to look for issues
+    :param ent_type: whether to retrieve issues or PRs (as issues)
+    :param since: the timestamp after which to retrieve
+    :return: the list of all retrieved entities
+    """
+
+    entity_info = ENTITY_INFO[ent_type]
+    all_entities = []
+    for repo_name in repo_names:
+        log.info(f"Looking for {entity_info.display_name}s in {org_name}/{repo_name}")
+        entities = gh.search_issues(
+            query="",
+            sort="updated",
+            order="desc",
+            **{
+                "repo": f"{org_name}/{repo_name}",
+                "is": ent_type,
+                "state": "open",
+                "created": f">={since.isoformat()}",
+            },
+        )
+        all_entities += list(entities)
+
+    log.info(f"Found {len(all_entities)} new {entity_info.display_name}s created")
+    return all_entities
+
+
+if __name__ == "__main__":
+    configure_logger()
+
+    args = parser.parse_args()
+
+    log.debug(f"Entity type: {args.entity_type}")
+    log.debug(f"Project number: {args.proj_number}")
+    log.debug(f"Target column name: {args.target_col_name}")
+    log.debug(f"Time period: {args.period}m")
+
+    since = datetime.datetime.utcnow() - datetime.timedelta(minutes=args.period)
+
+    github_info = get_data("github.yml")
+    org_name = github_info["org"]
+    log.info(f"Organization name: {org_name}")
+    repo_names = github_info["repos"].values()
+    log.info(f"Repository names: {', '.join(repo_names)}")
+
+    gh = get_client()
+    org = gh.get_organization(org_name)
+
+    entity_type = args.entity_type
+    entity_info = ENTITY_INFO[entity_type]
+    new_entities: list[Issue] = get_new_issues(
+        gh=gh,
+        org_name=org_name,
+        repo_names=repo_names,
+        ent_type=entity_type,
+        since=since,
+    )
+    if entity_type == "pr":
+        new_entities: list[PullRequest] = [
+            entity.as_pull_request() for entity in new_entities
+        ]
+
+    proj = get_org_project(org=org, proj_number=args.proj_number)
+    log.info(f"Found project: {proj.name}")
+    target_column = get_project_column(proj=proj, col_name=args.target_col_name)
+    log.debug("Found target column")
+
+    for entity in new_entities:
+        log.info(f"Creating card for {entity_info.display_name} {entity.number}")
+        try:
+            target_column.create_card(
+                content_id=entity.id,
+                content_type=entity_info.content_type,
+            )
+        except GithubException as ex:
+            if "Project already has the associated" in str(ex):
+                log.warning(f"Card already exists")
+            else:
+                log.error(
+                    f"Failed to create card for {entity_info.display_name} {entity.number}"
+                )

python/shared/log.py

@@ -0,0 +1,22 @@
+import logging
+import os
+
+
+def configure_logger():
+    """
+    Configures the logging module to
+    - change the log level names to lowercase
+    - set the formatter that works with GitHub logging commands
+    - set the default log level to LOGGING_LEVEL environment variable
+    """
+
+    logging.addLevelName(logging.CRITICAL, "critical")
+    logging.addLevelName(logging.ERROR, "error")
+    logging.addLevelName(logging.WARNING, "warning")
+    logging.addLevelName(logging.INFO, "info")
+    logging.addLevelName(logging.DEBUG, "debug")
+
+    logging.basicConfig(
+        format="::%(levelname)s::[%(name)s] %(message)s",
+        level=int(os.getenv("LOGGING_LEVEL", logging.DEBUG)),
+    )

python/shared/project.py

@@ -0,0 +1,47 @@
+import logging
+
+from github import (
+    Organization,
+    Project,
+    ProjectColumn,
+)
+
+log = logging.getLogger(__name__)
+
+
+def get_org_project(org: Organization, proj_number: int) -> Project:
+    """
+    Get the project with the given number in the given organization.
+
+    :param org: the organization in which to find the project
+    :param proj_number: the number of the project to find in the organization
+    :return: the project being searched for
+    :raise: ValueError if no project found with given number
+    """
+
+    log.info(f"Getting project {proj_number} in org {org.name}")
+    projects = org.get_projects()
+    project = next(proj for proj in projects if proj.number == proj_number)
+    if project is None:
+        log.error(f"No project was found with number {proj_number}.")
+        raise ValueError(f"Project not found")
+    return project
+
+
+def get_project_column(proj: Project, col_name: str) -> ProjectColumn:
+    """
+    Get the project column with the given name in the given project.
+
+    :param proj: the project in which to find the column
+    :param col_name: the name of the project column to find in the project
+    :return: the project column being searched for
+    :raise: ValueError if no project column found with given name
+    """
+
+    log.info(f"Getting column {col_name} in project {proj.name}")
+    columns = proj.get_columns()
+    column = next(col for col in columns if col.name == col_name)
+    if column is None:
+        log.error(f"No column was found with name {col_name}.")
+        raise ValueError(f"Column not found")
+    return column