ci: release note generation to handle breaking changes

.github/release-note-generation/golden/java-vertexai/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## 0.50.0 (2023-06-08)
 
+### ⚠ BREAKING CHANGES
+
+* An existing field `entry` is removed from message `some.service.SearchEntriesResult`
+* An existing field `display_name` is removed from message `some.service.SearchEntriesResult`
+
 ### Features
 
 * some handwritten changes ([#1234](https://github.com/googleapis/google-cloud-java/issues/1234)) ([a268c80](https://github.com/googleapis/google-cloud-java/commit/a268c8016262a7a5a13be6a9983294f83d1ecc3f))

.github/release-note-generation/split_release_note.py

@@ -17,9 +17,12 @@
 from pathlib import Path
 import json
 import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
 
 
 class LibraryModule:
+    """A module that represents one of the top-level library folders in
+    the google-cloud-java repository"""
     def __init__(self, path: Path, api_name: str, version: str,
         changelog: Path):
         self.path = path
@@ -34,6 +37,20 @@ def __repr__(self):
         return f"LibraryModule({self.path}, {self.api_name}, {self.version}, {self.changelog})"
 
 
+@dataclass
+class ChangesOnApi:
+    """Per-module changes, categorized as breaking changes, features, bug fixes,
+    and dependency upgrades"""
+    breaking_changes: [str] = field(default_factory=list)
+    features: [str] = field(default_factory=list)
+    bug_fixes: [str] = field(default_factory=list)
+    dependency_upgrades: [str] = field(default_factory=list)
+
+    def __len__(self):
+        return len(self.breaking_changes) + len(self.features) \
+            + len(self.bug_fixes) + len(self.dependency_upgrades)
+
+
 POM_NAMESPACES = {'mvn': 'http://maven.apache.org/POM/4.0.0'}
 
 
@@ -80,17 +97,46 @@ def detect_modules(root_directory: Path):
     return modules
 
 
-# Returns the dictionary from api name to a list of changelogs
+CHANGELOG_HEADER_MARK = '# Changelog'
+BREAKING_CHANGE_SECTION = '⚠ BREAKING CHANGES'
+FEATURES_SECTION = 'Features'
+BUG_FIXES_SECTION = 'Bug Fixes'
+DEPENDENCIES_SECTION = 'Dependencies'
+
+
+# Returns the dictionary from api name to ChangesOnApi
 def group_changes_by_api(main_changes: [str]):
-    api_to_changelog = defaultdict(list)
+    api_to_changelog = defaultdict(ChangesOnApi)
+    section = None
     for changelog in main_changes:
+        if changelog.endswith(BREAKING_CHANGE_SECTION):
+            section = BREAKING_CHANGE_SECTION
+            continue
+        if changelog.endswith(FEATURES_SECTION):
+            section = FEATURES_SECTION
+            continue
+        if changelog.endswith(BUG_FIXES_SECTION):
+            section = BUG_FIXES_SECTION
+            continue
+        if changelog.endswith(DEPENDENCIES_SECTION):
+            section = DEPENDENCIES_SECTION
+            continue
+
         match = re.search(r'\* \[(.+?)] (.+)', changelog)
         if match:
             api_name = match.group(1)
             note = match.group(2)
-            api_to_changelog[api_name].append(note)
+            if section == BREAKING_CHANGE_SECTION:
+                api_to_changelog[api_name].breaking_changes.append(note)
+            elif section == FEATURES_SECTION:
+                api_to_changelog[api_name].features.append(note)
+            elif section == BUG_FIXES_SECTION:
+                api_to_changelog[api_name].bug_fixes.append(note)
+            elif section == DEPENDENCIES_SECTION:
+                api_to_changelog[api_name].dependencies.append(note)
     return api_to_changelog
 
+
 def find_repo_wide_dependency_changes(main_changes: [str]):
     repo_wide_changes = []
     for changelog in main_changes:
@@ -102,29 +148,36 @@ def find_repo_wide_dependency_changes(main_changes: [str]):
     return repo_wide_changes
 
 
-CHANGELOG_HEADER_MARK = '# Changelog'
-
-
 def create_changelog_entry(current_date: str, module: LibraryModule,
-    changelog_lines: [str], dependency_changes: [str]):
+                           changes: ChangesOnApi):
     changelog_entry = f'## {module.version} ({current_date})\n\n'
-    if changelog_lines:
-        changelog_entry += '### Features\n\n'
-        for line in changelog_lines:
+    if changes.breaking_changes:
+        changelog_entry += f'### {BREAKING_CHANGE_SECTION}\n\n'
+        for line in changes.breaking_changes:
+            changelog_entry += f'* {line}\n'
+        changelog_entry += '\n'
+    if changes.features:
+        changelog_entry += f'### {FEATURES_SECTION}\n\n'
+        for line in changes.features:
+            changelog_entry += f'* {line}\n'
+        changelog_entry += '\n'
+    if changes.bug_fixes:
+        changelog_entry += f'### {BUG_FIXES_SECTION}\n\n'
+        for line in changes.bug_fixes:
             changelog_entry += f'* {line}\n'
         changelog_entry += '\n'
-    if dependency_changes:
-        changelog_entry += "### Dependencies\n\n"
-        for line in dependency_changes:
+    if changes.dependency_upgrades:
+        changelog_entry += f"### {DEPENDENCIES_SECTION}\n\n"
+        for line in changes.dependency_upgrades:
             changelog_entry += f'* {line}\n'
 
-    if len(changelog_lines) == 0 and len(dependency_changes) == 0:
+    if len(changes) == 0:
         changelog_entry += '* No change\n'
     return changelog_entry
 
 
 def write_changelog(current_date: str, module: LibraryModule,
-    changelog_entries: [str], dependency_changes: [str]):
+                    changelog_entries: [str]):
     changelog_file = module.changelog
     if changelog_file.exists():
         with open(changelog_file, 'r') as file:
@@ -136,8 +189,7 @@ def write_changelog(current_date: str, module: LibraryModule,
     if re.search(f'## {module.version}', changelog_content):
         return
 
-    entry = create_changelog_entry(current_date, module, changelog_entries,
-                                   dependency_changes)
+    entry = create_changelog_entry(current_date, module, changelog_entries)
     replaced = changelog_content.replace(CHANGELOG_HEADER_MARK,
                                          f'{CHANGELOG_HEADER_MARK}'
                                          f'\n\n{entry}')
@@ -166,21 +218,24 @@ def main():
             break
 
 
-    # Step 2: Detects target modules by .Owlbot-hermetic.yaml for api-name: field.
+    # Step 2: Detects target modules by .Owlbot-hermetic.yaml for
+    # the "api-name:" field.
     root_directory = sys.argv[2]
     modules = detect_modules(Path(root_directory))
     api_to_modules = {module.api_name: module for module in modules}
 
     # Step 3: Splits the changelog to ~100 modules
     api_to_changelog_entries = group_changes_by_api(main_changes)
-    dependency_change_entries = find_repo_wide_dependency_changes(main_changes)
+    global_dependency_change_entries = find_repo_wide_dependency_changes(main_changes)
 
     # Step 4: Writes the changelog entry to the CHANGELOG.md files in the
-    # modules
+    # modules. The global dependency changes appear in each module.
     for module in modules:
-        changelog_entries = api_to_changelog_entries.get(module.api_name, [])
-        write_changelog(release_date, module, changelog_entries,
-                        dependency_change_entries)
+        changes = api_to_changelog_entries.get(module.api_name,
+                                               ChangesOnApi())
+        if global_dependency_change_entries:
+            changes.dependency_upgrades.extend(global_dependency_change_entries)
+        write_changelog(release_date, module, changes)
 
 
 if __name__ == '__main__':

.github/release-note-generation/testdata/main_release_note.txt

@@ -6,6 +6,10 @@
 
 ## [1.13.0](https://github.com/googleapis/google-cloud-java/compare/v1.12.0...v1.13.0) (2023-06-08)
 
+### ⚠ BREAKING CHANGES
+
+* [vertexai] An existing field `entry` is removed from message `some.service.SearchEntriesResult`
+* [vertexai] An existing field `display_name` is removed from message `some.service.SearchEntriesResult`
 
 ### Features
 

.github/release-note-generation/unit_test.py

@@ -2,7 +2,7 @@
 
 # Unit tests for split_release_note.py
 
-from split_release_note import LibraryModule, create_changelog_entry, group_changes_by_api
+from split_release_note import LibraryModule, create_changelog_entry, group_changes_by_api, ChangesOnApi
 from pathlib import Path
 
 dummy_module = LibraryModule(
@@ -14,12 +14,16 @@
 class TestCase(unittest.TestCase):
 
     def test_create_changelog_entry(self):
+        changes = ChangesOnApi(
+            features=['Add support for disabling Pod overprovisioning',
+                      'Enhanced query generation performance'],
+            dependency_upgrades=[
+                'update google-cloud-shared-dependencies to 1.2.3']
+        )
         entry = create_changelog_entry(
             '2023-06-10',
             dummy_module,
-            ['Add support for disabling Pod overprovisioning',
-             'Enhanced query generation performance'],
-            ['update google-cloud-shared-dependencies to 1.2.3']
+            changes
         )
         self.assertEqual(entry, f'''## 1.2.3 (2023-06-10)
 
@@ -34,11 +38,14 @@ def test_create_changelog_entry(self):
 ''')
 
     def test_create_changelog_entry_only_deps(self):
+        dep_changes = ChangesOnApi(
+            dependency_upgrades=[
+                'update google-cloud-shared-dependencies to 1.2.3']
+        )
         entry = create_changelog_entry(
             '2023-06-10',
             dummy_module,
-            [],
-            ['update google-cloud-shared-dependencies to 1.2.3']
+            dep_changes,
         )
         self.assertEqual(entry, f'''## 1.2.3 (2023-06-10)
 
@@ -48,11 +55,11 @@ def test_create_changelog_entry_only_deps(self):
 ''')
 
     def test_create_changelog_entry_empty(self):
+        empty_changes = ChangesOnApi()
         entry = create_changelog_entry(
             '2023-06-10',
             dummy_module,
-            [],
-            []
+            empty_changes
         )
         self.assertEqual(entry, f'''## 1.2.3 (2023-06-10)
 
@@ -69,7 +76,7 @@ def test_group_changes_by_api(self):
         self.assertEqual(set(changes_by_api.keys()),
                          {'foo-api', 'bar-api'})
         self.assertEqual(changes_by_api['bar-api'],
-                         ['This is Change B'])
+                         ChangesOnApi(features=['This is Change B']))
         self.assertEqual(changes_by_api.get('nonexistent',
                                      ['No change']),
                          ['No change'])