From 849a94b5a40c51a7344d158a02a39449ebd720f2 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 15 Nov 2021 11:40:33 +0100 Subject: [PATCH] Add script to generate issue for status of testing of the rc (#19247) --- dev/ISSUE_TEMPLATE.md.jinja2 | 21 +++ dev/README_RELEASE_AIRFLOW.md | 29 ++++ dev/prepare_release_issue.py | 286 ++++++++++++++++++++++++++++++++++ 3 files changed, 336 insertions(+) create mode 100644 dev/ISSUE_TEMPLATE.md.jinja2 create mode 100755 dev/prepare_release_issue.py diff --git a/dev/ISSUE_TEMPLATE.md.jinja2 b/dev/ISSUE_TEMPLATE.md.jinja2 new file mode 100644 index 0000000000000..35be89265299a --- /dev/null +++ b/dev/ISSUE_TEMPLATE.md.jinja2 @@ -0,0 +1,21 @@ + + +We have a kind request for all the contributors to the latest [Apache Airflow RC {{version}}](https://pypi.org/project/apache-airflow/{{version}}/). + +Could you please help us to test the RC versions of Airflow? + +Please let us know in the comment if the issue is addressed in the latest RC. + +{% for pr_number in pr_list %} + {%- set pr = pull_requests[pr_number] -%} +- [ ] [{{ pr.title }} (#{{ pr.number }})]({{ pr.html_url }}): {{ user_logins[pr_number] }} + {%- if linked_issues[pr_number] %} + Linked issues: + {%- for linked_issue in linked_issues[pr_number] %} + - [{{ linked_issue.title }} (#{{ linked_issue.number }})]({{ linked_issue.html_url }}) + {%- endfor %} + {%- endif %} +{% endfor %} + +Thanks to all who contributed to the release (probably not a complete list!): +{{ all_user_logins }} diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 2edbc207aa9a1..99e077dbcfff2 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -27,6 +27,7 @@ - [[\Optional\] Prepare new release branches and cache](#%5Coptional%5C-prepare-new-release-branches-and-cache) - [Prepare PyPI convenience "snapshot" packages](#prepare-pypi-convenience-snapshot-packages) - [Prepare production Docker Image](#prepare-production-docker-image) + - [Prepare issue for testing status of rc](#prepare-issue-for-testing-status-of-rc) - [Prepare Vote email on the Apache Airflow release candidate](#prepare-vote-email-on-the-apache-airflow-release-candidate) - [Verify the release candidate by PMCs](#verify-the-release-candidate-by-pmcs) - [SVN check](#svn-check) @@ -337,6 +338,32 @@ Production Docker images should be manually prepared and pushed by the release m This will wipe Breeze cache and docker-context-files in order to make sure the build is "clean". It also performs image verification before pushing the images. +## Prepare issue for testing status of rc + +For now this part works for bugfix releases only, for major/minor ones we will experiment and +see if there is a way to only extract important/not tested bugfixes and high-level changes to +make the process manageable. + + +Create an issue for testing status of the RC (PREVIOUS_RELEASE should be the previous release version +(for example 2.1.0). + +```shell script +cat < \ + --current-release ${VERSION} + +``` + +Copy the URL of the issue. + ## Prepare Vote email on the Apache Airflow release candidate - Use the dev/airflow-jira script to generate a list of Airflow JIRAs that were closed in the release. @@ -363,6 +390,8 @@ until Monday, October 11, 2021 at 4:00 pm UTC, or until 3 binding +1 votes have https://www.timeanddate.com/worldclock/fixedtime.html?msg=8&iso=20211011T1600&p1=1440 +Status of testing of the release is kept in TODO:URL_OF_THE_ISSUE_HERE + Consider this my (binding) +1. Airflow ${VERSION} is available at: diff --git a/dev/prepare_release_issue.py b/dev/prepare_release_issue.py new file mode 100755 index 0000000000000..c98684d19d200 --- /dev/null +++ b/dev/prepare_release_issue.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import os +import re +import subprocess +import textwrap +from collections import defaultdict +from typing import Any, Dict, List, NamedTuple, Optional, Set + +import click +from github import Github, Issue, PullRequest, UnknownObjectException +from rich.console import Console +from rich.progress import Progress + +logger = logging.getLogger(__name__) + +console = Console(width=400, color_system="standard") + +MY_DIR_PATH = os.path.dirname(__file__) +SOURCE_DIR_PATH = os.path.abspath(os.path.join(MY_DIR_PATH, os.pardir)) +PR_PATTERN = re.compile(r".*\(#([0-9]+)\)") +ISSUE_MATCH_IN_BODY = re.compile(r" #([0-9]+)[^0-9]") + + +@click.group(context_settings={'help_option_names': ['-h', '--help'], 'max_content_width': 500}) +def cli(): + ... + + +option_verbose = click.option( + "--verbose", + is_flag=True, + help="Print verbose information about performed steps", +) + +option_previous_release = click.option( + "--previous-release", + type=str, + required=True, + help="commit reference (for example hash or tag) of the previous release.", +) + +option_current_release = click.option( + "--current-release", + type=str, + required=True, + help="commit reference (for example hash or tag) of the current release.", +) + +option_github_token = click.option( + "--github-token", + type=str, + required=True, + help=textwrap.dedent( + """ + Github token used to authenticate. + You can set omit it if you have GITHUB_TOKEN env variable set + Can be generated with: + https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status""" + ), + envvar='GITHUB_TOKEN', +) + +option_excluded_pr_list = click.option( + "--excluded-pr-list", type=str, default='', help="Coma-separated list of PRs to exclude from the issue." +) + +option_limit_pr_count = click.option( + "--limit-pr-count", + type=int, + default=None, + help="Limit PR count processes (useful for testing small subset of PRs).", +) + + +def get_git_log_command( + verbose: bool, from_commit: Optional[str] = None, to_commit: Optional[str] = None +) -> List[str]: + """ + Get git command to run for the current repo from the current folder (which is the package folder). + :param verbose: whether to print verbose info while getting the command + :param from_commit: if present - base commit from which to start the log from + :param to_commit: if present - final commit which should be the start of the log + :return: git command to run + """ + git_cmd = [ + "git", + "log", + "--pretty=format:%H %h %cd %s", + "--date=short", + ] + if from_commit and to_commit: + git_cmd.append(f"{from_commit}...{to_commit}") + elif from_commit: + git_cmd.append(from_commit) + git_cmd.extend(['--', '.']) + if verbose: + console.print(f"Command to run: '{' '.join(git_cmd)}'") + return git_cmd + + +class Change(NamedTuple): + """Stores details about commits""" + + full_hash: str + short_hash: str + date: str + message: str + message_without_backticks: str + pr: Optional[int] + + +def get_change_from_line(line: str): + split_line = line.split(" ", maxsplit=3) + message = split_line[3] + pr = None + pr_match = PR_PATTERN.match(message) + if pr_match: + pr = pr_match.group(1) + return Change( + full_hash=split_line[0], + short_hash=split_line[1], + date=split_line[2], + message=message, + message_without_backticks=message.replace("`", "'").replace("&39;", "'"), + pr=int(pr) if pr else None, + ) + + +def get_changes(verbose: bool, previous_release: str, current_release: str) -> List[Change]: + change_strings = subprocess.check_output( + get_git_log_command(verbose, from_commit=previous_release, to_commit=current_release), + cwd=SOURCE_DIR_PATH, + universal_newlines=True, + ) + return [get_change_from_line(line) for line in change_strings.split("\n")] + + +def render_template( + template_name: str, + context: Dict[str, Any], + autoescape: bool = True, + keep_trailing_newline: bool = False, +) -> str: + """ + Renders template based on it's name. Reads the template from _TEMPLATE.md.jinja2 in current dir. + :param template_name: name of the template to use + :param context: Jinja2 context + :param autoescape: Whether to autoescape HTML + :param keep_trailing_newline: Whether to keep the newline in rendered output + :return: rendered template + """ + import jinja2 + + template_loader = jinja2.FileSystemLoader(searchpath=MY_DIR_PATH) + template_env = jinja2.Environment( + loader=template_loader, + undefined=jinja2.StrictUndefined, + autoescape=autoescape, + keep_trailing_newline=keep_trailing_newline, + ) + template = template_env.get_template(f"{template_name}_TEMPLATE.md.jinja2") + content: str = template.render(context) + return content + + +def print_issue_content( + current_release: str, + pull_requests: Dict[int, PullRequest.PullRequest], + linked_issues: Dict[int, List[Issue.Issue]], + users: Dict[int, Set[str]], +): + pr_list = list(pull_requests.keys()) + pr_list.sort() + user_logins: Dict[int, str] = {pr: "@" + " @".join(users[pr]) for pr in users} + all_users: Set[str] = set() + for user_list in users.values(): + all_users.update(user_list) + all_user_logins = "@" + " @".join(all_users) + content = render_template( + template_name='ISSUE', + context={ + 'version': current_release, + 'pr_list': pr_list, + 'pull_requests': pull_requests, + 'linked_issues': linked_issues, + 'users': users, + 'user_logins': user_logins, + 'all_user_logins': all_user_logins, + }, + autoescape=False, + keep_trailing_newline=True, + ) + print(content) + + +@cli.command() +@option_github_token +@option_previous_release +@option_current_release +@option_excluded_pr_list +@option_verbose +@option_limit_pr_count +def generate_issue_content( + github_token: str, + previous_release: str, + current_release: str, + excluded_pr_list: str, + verbose: bool, + limit_pr_count: Optional[int], +): + if excluded_pr_list: + excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")] + else: + excluded_prs = [] + changes = get_changes(verbose, previous_release, current_release) + prs = list( + filter(lambda pr: pr is not None and pr not in excluded_prs, [change.pr for change in changes]) + ) + g = Github(github_token) + repo = g.get_repo("apache/airflow") + pull_requests: Dict[int, PullRequest.PullRequest] = {} + linked_issues: Dict[int, List[Issue.Issue]] = defaultdict(lambda: []) + users: Dict[int, Set[str]] = defaultdict(lambda: set()) + count_prs = len(prs) + if limit_pr_count: + count_prs = limit_pr_count + with Progress(console=console) as progress: + task = progress.add_task(f"Retrieving {count_prs} PRs ", total=count_prs) + for i in range(count_prs): + pr_number = prs[i] + progress.console.print( + f"Retrieving PR#{pr_number}: " f"https://github.com/apache/airflow/pull/{pr_number}" + ) + try: + pr = repo.get_pull(pr_number) + except UnknownObjectException: + # Fallback to issue if PR not found + try: + pr = repo.get_issue(pr_number) # (same fields as PR) + except UnknownObjectException: + console.print(f"[red]The PR #{pr_number} could not be found[/]") + continue + pull_requests[pr_number] = pr + # GitHub does not have linked issues in PR - but we quite rigorously add Fixes/Closes + # Relate so we can find those from the body + if pr.body: + body = pr.body.replace("\n", " ").replace("\r", " ") + for issue_match in ISSUE_MATCH_IN_BODY.finditer(body): + linked_issue_number = int(issue_match.group(1)) + progress.console.print( + f"Retrieving Linked issue PR#{linked_issue_number}: " + f"https://github.com/apache/airflow/issue/{linked_issue_number}" + ) + try: + linked_issues[pr_number].append(repo.get_issue(linked_issue_number)) + except UnknownObjectException: + progress.console.print( + f"Failed to retrieve linked issue #{linked_issue_number}: Unknown Issue" + ) + users[pr_number].add(pr.user.login) + for linked_issue in linked_issues[pr_number]: + users[pr_number].add(linked_issue.user.login) + progress.advance(task) + print_issue_content(current_release, pull_requests, linked_issues, users) + + +if __name__ == "__main__": + cli()