diff --git a/scripts/release-notes.py b/scripts/release-notes.py index 69398692e0bf..ba9d9b38e524 100755 --- a/scripts/release-notes.py +++ b/scripts/release-notes.py @@ -64,6 +64,7 @@ 'Amruta': "Amruta Ranade", 'yuzefovich': "Yahor Yuzefovich", 'madhavsuresh': "Madhav Suresh", + 'Richard Loveland': "Rich Loveland", } # FIXME(knz): This too. @@ -75,15 +76,19 @@ "Andrei Matei", "Andrew Couch", "Andrew Kimball", + "Andrew Werner", "Andy Woods", "Arjun Narayan", "Ben Darnell", + "Bilal Akthar", "Bob Vawter", "Bram Gruneir", + "Celia La", "Daniel Harrison", "David Taylor", "Diana Hsieh", "Emmanuel Sales", + "Erik Trinh", "Jesse Seldess", "Jessica Edwards", "Joseph Lowinske", @@ -92,6 +97,7 @@ "Justin Jaffray", "Kuan Luo", "Lauren Hirata", + "Lucy Zhang", "Madhav Suresh", "Marc Berhault", "Masha Schneider", @@ -123,7 +129,7 @@ # Section titles for release notes. relnotetitles = { - 'cli change': "Command line changes", + 'cli change': "Command-line changes", 'sql change': "SQL language changes", 'admin ui change': "Admin UI changes", 'general change': "General changes", @@ -214,6 +220,12 @@ help="omit the email sign-up and downloads section") parser.add_option("--hide-header", action="store_true", dest="hide_header", default=False, help="omit the title and date header") +parser.add_option("--exclude-from", dest="exclude_from_commit", + help="exclude history starting after COMMIT. Note: COMMIT itself is excluded.", metavar="COMMIT") +parser.add_option("--exclude-until", dest="exclude_until_commit", + help="exclude history ending at COMMIT", metavar="COMMIT") +parser.add_option("--one-line", dest="one_line", action="store_true", default=False, + help="unwrap release notes on a single line") (options, args) = parser.parse_args() @@ -228,20 +240,41 @@ repo = Repo('.') heads = repo.heads -try: - firstCommit = repo.commit(options.from_commit) -except: - print("Unable to find the first commit of the range.", file=sys.stderr) - print("No ref named %s." % options.from_commit, file=sys.stderr) - exit(0) +def reformat_note(note_lines): + sep = '\n' + if options.one_line: + sep = ' ' + return sep.join(note_lines).strip() +# Check that pull_ref_prefix is valid +testrefname = "%s/1" % pull_ref_prefix try: - commit = repo.commit(options.until_commit) + repo.commit(testrefname) except: - print("Unable to find the last commit of the range.", file=sys.stderr) - print("No ref named %s." % options.until_commit, file=sys.stderr) - exit(0) + print("Unable to find pull request refs at %s." % pull_ref_prefix, file=sys.stderr) + print("Is your repo set up to fetch them? Try adding", file=sys.stderr) + print(" fetch = +refs/pull/*/head:%s/*" % pull_ref_prefix, file=sys.stderr) + print("to the GitHub remote section of .git/config.", file=sys.stderr) + exit(1) + +def find_commits(from_commit_ref, until_commit_ref): + try: + firstCommit = repo.commit(from_commit_ref) + except: + print("Unable to find the first commit of the range.", file=sys.stderr) + print("No ref named %s." % from_commit_ref, file=sys.stderr) + exit(1) + + try: + commit = repo.commit(until_commit_ref) + except: + print("Unable to find the last commit of the range.", file=sys.stderr) + print("No ref named %s." % until_commit_ref, file=sys.stderr) + exit(1) + return firstCommit, commit + +firstCommit, commit = find_commits(options.from_commit, options.until_commit) if commit == firstCommit: print("Commit range is empty!", file=sys.stderr) print(parser.get_usage(), file=sys.stderr) @@ -252,16 +285,12 @@ print("Note: the first commit is excluded. Use e.g.: --from --until ", file=sys.stderr) exit(0) -# Check that pull_ref_prefix is valid -testrefname = "%s/1" % pull_ref_prefix -try: - repo.commit(testrefname) -except: - print("Unable to find pull request refs at %s." % pull_ref_prefix, file=sys.stderr) - print("Is your repo set up to fetch them? Try adding", file=sys.stderr) - print(" fetch = +refs/pull/*/head:%s/*" % pull_ref_prefix, file=sys.stderr) - print("to the GitHub remote section of .git/config.", file=sys.stderr) - exit(0) +excludedFirst, excludedLast = None, None +if options.exclude_from_commit or options.exclude_until_commit: + if not options.exclude_from_commit or not options.exclude_until_commit: + print("Both -xf and -xt must be specified, or not at all.") + exit(1) + excludedFirst, excludedLast = find_commits(options.exclude_from_commit, options.exclude_until_commit) ### Reading data from repository ### @@ -270,51 +299,33 @@ def identify_commit(commit): commit.hexsha, commit.message.split('\n',1)[0], datetime.datetime.fromtimestamp(commit.committed_date).ctime()) -# Is the first commit reachable from the current one? -base = repo.merge_base(firstCommit, commit) -if len(base) == 0: - print("error: %s:%s\nand %s:%s\nhave no common ancestor" % ( - options.from_commit, identify_commit(firstCommit), - options.until_commit, identify_commit(commit)), file=sys.stderr) - exit(1) -commonParent = base[0] -if firstCommit != commonParent: - print("warning: %s:%s\nis not an ancestor of %s:%s!" % ( - options.from_commit, identify_commit(firstCommit), - options.until_commit, identify_commit(commit)), file=sys.stderr) - print(file=sys.stderr) - ageindays = int((firstCommit.committed_date - commonParent.committed_date)/86400) - prevlen = sum((1 for x in repo.iter_commits(commonParent.hexsha + '...' + firstCommit.hexsha))) - print("The first common ancestor is %s" % identify_commit(commonParent), file=sys.stderr) - print("which is %d commits older than %s:%s\nand %d days older. Using that as origin." %\ - (prevlen, options.from_commit, identify_commit(firstCommit), ageindays), file=sys.stderr) - print(file=sys.stderr) - firstCommit = commonParent - options.from_commit = commonParent.hexsha - -print("Changes from\n%s\nuntil\n%s" % (identify_commit(firstCommit), identify_commit(commit)), file=sys.stderr) - -release_notes = {} -missing_release_notes = [] - -def collect_authors(commit): - authors = set() - author = author_aliases.get(commit.author.name, commit.author.name) - if author != 'GitHub': - authors.add(author) - author = author_aliases.get(commit.committer.name, commit.committer.name) - if author != 'GitHub': - authors.add(author) - for m in coauthor.finditer(commit.message): - aname = m.group('name').strip() - author = author_aliases.get(aname, aname) - authors.add(author) - return authors - - -def extract_release_notes(pr, title, commit): - authors = collect_authors(commit) - +def check_reachability(firstCommit, commit): + # Is the first commit reachable from the current one? + base = repo.merge_base(firstCommit, commit) + if len(base) == 0: + print("error: %s:%s\nand %s:%s\nhave no common ancestor" % ( + options.from_commit, identify_commit(firstCommit), + options.until_commit, identify_commit(commit)), file=sys.stderr) + exit(1) + commonParent = base[0] + if firstCommit != commonParent: + print("warning: %s:%s\nis not an ancestor of %s:%s!" % ( + options.from_commit, identify_commit(firstCommit), + options.until_commit, identify_commit(commit)), file=sys.stderr) + print(file=sys.stderr) + ageindays = int((firstCommit.committed_date - commonParent.committed_date)/86400) + prevlen = sum((1 for x in repo.iter_commits(commonParent.hexsha + '...' + firstCommit.hexsha))) + print("The first common ancestor is %s" % identify_commit(commonParent), file=sys.stderr) + print("which is %d commits older than %s:%s\nand %d days older. Using that as origin." %\ + (prevlen, options.from_commit, identify_commit(firstCommit), ageindays), file=sys.stderr) + print(file=sys.stderr) + firstCommit = commonParent + return firstCommit, commit + +firstCommit, commit = check_reachability(firstCommit, commit) +options.from_commit = firstCommit.hexsha + +def extract_release_notes(commit): msglines = commit.message.split('\n') curnote = [] innote = False @@ -353,7 +364,7 @@ def extract_release_notes(pr, title, commit): # We have a release note boundary. If we were collecting a # note already, complete it. if innote: - notes.append((cat, curnote)) + notes.append((cat, reformat_note(curnote))) curnote = [] innote = False @@ -386,14 +397,94 @@ def extract_release_notes(pr, title, commit): cat = cat_misspells[cat] if innote: - notes.append((cat, curnote)) + notes.append((cat, reformat_note(curnote))) + + return foundnote, notes + +spinner = itertools.cycle(['/', '-', '\\', '|']) +counter = 0 +def spin(): + global counter + # Display a progress bar + counter += 1 + if counter % 10 == 0: + if counter % 100 == 0: + print("\b..", end='', file=sys.stderr) + print("\b", end='', file=sys.stderr) + print(next(spinner), end='', file=sys.stderr) + sys.stderr.flush() + +def get_direct_history(firstCommit, lastCommit): + history = [] + for c in repo.iter_commits(firstCommit.hexsha + '..' + lastCommit.hexsha, first_parent = True): + history.append(c) + return history + +excluded_notes = set() +if excludedFirst is not None: + # + # Collect all the notes to exclude during collection below. + # + print("Collecting EXCLUDED release notes from\n%s\nuntil\n%s" % (identify_commit(excludedFirst), identify_commit(excludedLast)), file=sys.stderr) + + # First ensure that the loop below will terminate. + excludedFirst, excludedLast = check_reachability(excludedFirst, excludedLast) + # Collect all the merge points, so we can measure progress. + mergepoints = get_direct_history(excludedFirst, excludedLast) + + # Now collect all commits. + print("Collecting EXCLUDED release notes...", file=sys.stderr) + i = 0 + progress = 0 + lastTime = time.time() + for c in repo.iter_commits(excludedFirst.hexsha + '..' + excludedLast.hexsha): + progress = int(100. * float(i) / len(mergepoints)) + newTime = time.time() + if newTime >= lastTime + 5: + print("\b%d%%.." % progress, file=sys.stderr, end='') + lastTime = newTime + i += 1 + + spin() + # Collect the release notes in that commit. + _, notes = extract_release_notes(c) + for cat, note in notes: + excluded_notes.add((cat, note)) + + print("\b100%\n", file=sys.stderr) + +print("Collecting release notes from\n%s\nuntil\n%s" % (identify_commit(firstCommit), identify_commit(commit)), file=sys.stderr) + +release_notes = {} +missing_release_notes = [] + +def collect_authors(commit): + authors = set() + author = author_aliases.get(commit.author.name, commit.author.name) + if author != 'GitHub': + authors.add(author) + author = author_aliases.get(commit.committer.name, commit.committer.name) + if author != 'GitHub': + authors.add(author) + for m in coauthor.finditer(commit.message): + aname = m.group('name').strip() + author = author_aliases.get(aname, aname) + authors.add(author) + return authors + + +def process_release_notes(pr, title, commit): + authors = collect_authors(commit) + + foundnote, notes = extract_release_notes(commit) # At the end the notes will be presented in reverse order, because # we explore the commits in reverse order. However within 1 commit # the notes are in the correct order. So reverse them upfront here, # so that the 2nd reverse gets them in the right order again. for cat, note in reversed(notes): - completenote(commit, cat, note, authors, pr, title) + if (cat, note) not in excluded_notes: + completenote(commit, cat, note, authors, pr, title) missing_item = None if not foundnote: @@ -408,8 +499,7 @@ def makeitem(pr, prtitle, sha, authors): 'title': prtitle, 'note': None} -def completenote(commit, cat, curnote, authors, pr, title): - notemsg = '\n'.join(curnote).strip() +def completenote(commit, cat, notemsg, authors, pr, title): item = makeitem(pr, title, commit.hexsha[:shamin], authors) item['note'] = notemsg @@ -422,20 +512,6 @@ def completenote(commit, cat, curnote, authors, pr, title): individual_authors = set() allprs = set() -spinner = itertools.cycle(['/', '-', '\\', '|']) -counter = 0 - -def spin(): - global counter - # Display a progress bar - counter += 1 - if counter % 10 == 0: - if counter % 100 == 0: - print("\b..", end='', file=sys.stderr) - print("\b", end='', file=sys.stderr) - print(next(spinner), end='', file=sys.stderr) - sys.stderr.flush() - # This function groups and counts all the commits that belong to a particular PR. # Some description is in order regarding the logic here: it should visit all # commits that are on the PR and only on the PR. If there's some secondary @@ -524,10 +600,9 @@ def analyze_pr(merge, pr): missing_items = [] authors = set() ncommits = 0 - while len(commits_to_analyze) > 0: + for commit in repo.iter_commits(merge_base.hexsha + '..' + tip.hexsha): spin() - commit = commits_to_analyze.pop(0) if commit in seen_commits: # We may be seeing the same commit twice if a feature branch has # been forked in sub-branches. Just skip over what we've seen @@ -536,21 +611,12 @@ def analyze_pr(merge, pr): seen_commits.add(commit) if not commit.message.startswith("Merge"): - missing_item, prauthors = extract_release_notes(pr, note, commit) + missing_item, prauthors = process_release_notes(pr, note, commit) authors.update(prauthors) ncommits += 1 if missing_item is not None: missing_items.append(missing_item) - for parent in commit.parents: - if not repo.is_ancestor(parent, merge.parents[0]): - # We're not yet back on the main branch. Just continue digging. - commits_to_analyze.append(parent) - else: - # The parent is on the main branch. We're done digging. - # print("found merge parent, stopping. final authors", authors) - pass - if ncommits == len(missing_items): # None of the commits found had a release note. List them. for item in missing_items: @@ -586,7 +652,19 @@ def analyze_standalone_commit(commit): missing_release_notes.append(item) collect_item('#unknown', title, commit.hexsha[:shamin], 1, authors, commit.stats.total, commit.committed_date) -while commit != firstCommit: + +# Collect all the merge points so we can report progress. +mergepoints = get_direct_history(firstCommit, commit) +i = 0 +progress = 0 +lastTime = time.time() +for commit in mergepoints: + progress = int(100. * float(i) / len(mergepoints)) + newTime = time.time() + if newTime >= lastTime + 5: + print("\b.%d%%\n." % progress, file=sys.stderr, end='') + lastTime = newTime + i += 1 spin() ctime = datetime.datetime.fromtimestamp(commit.committed_date).ctime() @@ -601,9 +679,9 @@ def analyze_standalone_commit(commit): print(" \r%s (%s) " % (commit.hexsha[:shamin], ctime), end='', file=sys.stderr) analyze_standalone_commit(commit) - if len(commit.parents) == 0: - break - commit = commit.parents[0] + +print("\b\nAnalyzing authors...", file=sys.stderr) +sys.stderr.flush() allgroups = list(per_group_history.keys()) allgroups.sort(key=lambda x:x.lower()) @@ -644,7 +722,7 @@ def analyze_standalone_commit(commit): if not hideheader: print("---") print("title: What's New in", current_version) - print("toc: false") + print("toc: true") print("summary: Additions and changes in CockroachDB version", current_version, "since version", previous_version) print("---") print() @@ -681,8 +759,9 @@ def analyze_standalone_commit(commit): print("""### Docker image +{% include copy-clipboard.html %} ~~~shell -docker pull cockroachdb/cockroach:""" + current_version + """ +$ docker pull cockroachdb/cockroach:""" + current_version + """ ~~~ """) print() diff --git a/scripts/release-notes/common.sh b/scripts/release-notes/common.sh index 90e22bce5357..286d5ac01639 100644 --- a/scripts/release-notes/common.sh +++ b/scripts/release-notes/common.sh @@ -91,7 +91,7 @@ function test_end() { fi # Check the generated release notes. - (cd $t && $PYTHON $relnotescript --hide-header --hide-downloads-section --from initial --until master) >$t.notes.txt + (cd $t && $PYTHON $relnotescript --hide-header --hide-downloads-section --from initial --until master "$@") >$t.notes.txt if test -z "$rewrite"; then diff -u $t.notes.txt $t.notes.ref.txt else diff --git a/scripts/release-notes/test1.notes.ref.txt b/scripts/release-notes/test1.notes.ref.txt index d4c86d0f8d06..7d8a26dc6bec 100644 --- a/scripts/release-notes/test1.notes.ref.txt +++ b/scripts/release-notes/test1.notes.ref.txt @@ -1,4 +1,4 @@ -### Bug Fixes +### Bug fixes - Feature A [#1][#1] [20a33aaf1][20a33aaf1] @@ -8,7 +8,7 @@ - [#unknown][#unknown] [e3a1f2c94][e3a1f2c94] master update (test1) -### Doc Updates +### Doc updates Docs team: Please add these manually. diff --git a/scripts/release-notes/test2.notes.ref.txt b/scripts/release-notes/test2.notes.ref.txt index 9c688a49610a..0b5431a5b67e 100644 --- a/scripts/release-notes/test2.notes.ref.txt +++ b/scripts/release-notes/test2.notes.ref.txt @@ -1,4 +1,4 @@ -### Bug Fixes +### Bug fixes - Feature A [#1][#1] [beda26774][beda26774] - Feature B [#1][#1] [52d1235cc][52d1235cc] @@ -9,7 +9,7 @@ - [#unknown][#unknown] [f872999e8][f872999e8] master update (test2) -### Doc Updates +### Doc updates Docs team: Please add these manually. diff --git a/scripts/release-notes/test3.notes.ref.txt b/scripts/release-notes/test3.notes.ref.txt index 81441718af45..6dca361d0fce 100644 --- a/scripts/release-notes/test3.notes.ref.txt +++ b/scripts/release-notes/test3.notes.ref.txt @@ -1,4 +1,4 @@ -### Bug Fixes +### Bug fixes - Feature A [#1][#1] [6a84145e9][6a84145e9] - Feature C [#1][#1] [0ac9dc576][0ac9dc576] @@ -11,7 +11,7 @@ - [#unknown][#unknown] [4f4329fdc][4f4329fdc] master update (test3) -### Doc Updates +### Doc updates Docs team: Please add these manually. diff --git a/scripts/release-notes/test4.notes.ref.txt b/scripts/release-notes/test4.notes.ref.txt index 38b6d70e3266..ec22473f7780 100644 --- a/scripts/release-notes/test4.notes.ref.txt +++ b/scripts/release-notes/test4.notes.ref.txt @@ -1,4 +1,4 @@ -### Bug Fixes +### Bug fixes - Feature A release note 1 @@ -11,7 +11,7 @@ - Feature D [#1][#1] [662a10125][662a10125] - Feature E [#1][#1] [76bb9f090][76bb9f090] -### Doc Updates +### Doc updates Docs team: Please add these manually. diff --git a/scripts/release-notes/test5.notes.ref.txt b/scripts/release-notes/test5.notes.ref.txt index 7237360af62a..0d36c2a097bb 100644 --- a/scripts/release-notes/test5.notes.ref.txt +++ b/scripts/release-notes/test5.notes.ref.txt @@ -1,4 +1,4 @@ -### Bug Fixes +### Bug fixes - Feature A [#1][#1] [076393957][076393957] @@ -8,7 +8,7 @@ - [#2][#2] [8156afc96][8156afc96] PR title in need of release note (test5) -### Doc Updates +### Doc updates Docs team: Please add these manually. diff --git a/scripts/release-notes/test6.notes.ref.txt b/scripts/release-notes/test6.notes.ref.txt index 35fb8f3ad3b9..cc9dfc02c68b 100644 --- a/scripts/release-notes/test6.notes.ref.txt +++ b/scripts/release-notes/test6.notes.ref.txt @@ -1,4 +1,4 @@ -### Doc Updates +### Doc updates Docs team: Please add these manually. diff --git a/scripts/release-notes/test7.graph.ref.txt b/scripts/release-notes/test7.graph.ref.txt new file mode 100644 index 000000000000..0092bc0457ad --- /dev/null +++ b/scripts/release-notes/test7.graph.ref.txt @@ -0,0 +1,19 @@ +* 28245dacd01a6b6fc50245d108d6bd713361c1d7 Merge pull request #200 from foo/bar +|\ +| * 78b06acb8baec3e22dc47b4fd978405915400a97 merge pr canary +| * cb718dc99f4eb05d41de1da18fd027b0dc9f96bc Merge #2 +| |\ +| | * 6da808a53cf9a00fe72e12c2be9712d0f5d81dd4 backport A +| |/ +* | ba15054b5e8721606d2203aeb4241dd490696885 Merge pull request #100 from foo/bar +|\ \ +| * | 3c612877c5e9dac6a6d06c1161a275c2b2ab7848 merge pr canary +|/ / +* | 191b497b9b62a436e86cded6f98a199b11ec6e65 Merge #1 +|\ \ +| |/ +|/| +| * 57aa2eab630906aa41639fafdfe3cd11fad33bf4 feature B +| * 8b135a6544448ffb9d7977506e889b003f62a921 feature A +|/ +* 5d7ea53d1a3340e43c65ef950b3aa8516644e584 initial diff --git a/scripts/release-notes/test7.notes.ref.txt b/scripts/release-notes/test7.notes.ref.txt new file mode 100644 index 000000000000..4c6f9e66e086 --- /dev/null +++ b/scripts/release-notes/test7.notes.ref.txt @@ -0,0 +1,31 @@ +### Bug fixes + +- Feature B [#1][#1] [57aa2eab6][57aa2eab6] + +### Doc updates + +Docs team: Please add these manually. + +### Contributors + +This release includes 3 merged PRs by 1 author. +We would like to thank the following contributors from the CockroachDB community: + +- test7 + +### PRs merged by contributors + +- test7: + - 2018-04-22 [#200 ][#200 ] [28245dacd][28245dacd] (+ 0 - 0 ~ 0/ 0) PR title 2 alternate format (2 commits) + - 2018-04-22 [#100 ][#100 ] [ba15054b5][ba15054b5] (+ 0 - 0 ~ 0/ 0) PR title 1 alternate format + - 2018-04-22 [#1 ][#1 ] [191b497b9][191b497b9] (+ 0 - 0 ~ 0/ 0) PR title 1 (2 commits) + + +[#1]: https://github.com/cockroachdb/cockroach/pull/1 +[#100]: https://github.com/cockroachdb/cockroach/pull/100 +[#200]: https://github.com/cockroachdb/cockroach/pull/200 +[191b497b9]: https://github.com/cockroachdb/cockroach/commit/191b497b9 +[28245dacd]: https://github.com/cockroachdb/cockroach/commit/28245dacd +[57aa2eab6]: https://github.com/cockroachdb/cockroach/commit/57aa2eab6 +[ba15054b5]: https://github.com/cockroachdb/cockroach/commit/ba15054b5 + diff --git a/scripts/release-notes/test7.sh b/scripts/release-notes/test7.sh new file mode 100644 index 000000000000..4bd90be75aa0 --- /dev/null +++ b/scripts/release-notes/test7.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eux + +. common.sh + +t=test7 +relnotescript=${1:?} +rewrite=${2:-} + +test_init + +( + cd $t + init_repo + + git checkout -b release-branch + git checkout -b feature1 + make_change "feature A + +Release note (bug fix): feature A +" + make_change "feature B + +Release note (bug fix): feature B +" + tag_pr 1 + git checkout master + merge_pr feature1 1 "PR title 1" + + git checkout release-branch -b backport + make_change "backport A + +Release note (bug fix): feature A +" + tag_pr 2 + git checkout release-branch + merge_pr backport 2 "PR title 2" +) + +test_end --exclude-from initial --exclude-until release-branch