Add Python script to compile release notes

Add markdown parser 'mistune' in Pipfile Python script generates 'release-notes availability' table and URL file for verification Signed-off-by: Sachin Sahu <[email protected]>
SachinSahu431 · Nov 2, 2023 · bbf3ebe · bbf3ebe
1 parent 1515a1a
commit bbf3ebe
Show file tree

Hide file tree

Showing 6 changed files with 728 additions and 158 deletions.
diff --git a/Pipfile b/Pipfile
@@ -38,6 +38,7 @@ typed-ast = "~=1.5.4"
 zipp = "~=3.8.1"
 importlib-metadata = "~=4.12.0"
 ruamel-yaml = "~=0.17.21"
+mistune = "~=3.0.1"
 
 [dev-packages]
 

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/src/release_notes_automation/release_notes-2.9.0.md b/src/release_notes_automation/release_notes-2.9.0.md
diff --git a/src/release_notes_automation/release_notes_table-2.9.0.md b/src/release_notes_automation/release_notes_table-2.9.0.md
@@ -0,0 +1,21 @@
+#  OpenSearch CommitID(after 2022-07-26) & Release Notes info
+|          Repo           |    Branch    |CommitID|Commit Date|Release Notes Exists|                       Full Path                        |                                                                          URL                                                                           |
+|-------------------------|--------------|--------|-----------|--------------------|--------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|
+|alerting                 |[tags/2.9.0.0]|aefb268 |2023-07-13 |True                |opensearch-alerting.release-notes-2.9.0.0.md            |https://raw.githubusercontent.com/opensearch-project/alerting/2.9.0.0/release-notes/opensearch-alerting.release-notes-2.9.0.0.md                        |
+|anomaly-detection        |[tags/2.9.0.0]|62dd94f |2023-07-13 |True                |opensearch-anomaly-detection.release-notes-2.9.0.0.md   |https://raw.githubusercontent.com/opensearch-project/anomaly-detection/2.9.0.0/release-notes/opensearch-anomaly-detection.release-notes-2.9.0.0.md      |
+|asynchronous-search      |[tags/2.9.0.0]|68e7110 |2023-07-11 |True                |opensearch-asynchronous-search.release-notes-2.9.0.0.md |https://raw.githubusercontent.com/opensearch-project/asynchronous-search/2.9.0.0/release-notes/opensearch-asynchronous-search.release-notes-2.9.0.0.md  |
+|common-utils             |[tags/2.9.0.0]|cdd30e0 |2023-07-12 |True                |opensearch-common-utils.release-notes-2.9.0.0.md        |https://raw.githubusercontent.com/opensearch-project/common-utils/2.9.0.0/release-notes/opensearch-common-utils.release-notes-2.9.0.0.md                |
+|cross-cluster-replication|[tags/2.9.0.0]|7d5e071 |2023-07-18 |False               |                                                        |                                                                                                                                                        |
+|geospatial               |[tags/2.9.0.0]|f8df9a3 |2023-07-11 |True                |opensearch-geospatial.release-notes-2.9.0.0.md          |https://raw.githubusercontent.com/opensearch-project/geospatial/2.9.0.0/release-notes/opensearch-geospatial.release-notes-2.9.0.0.md                    |
+|index-management         |[tags/2.9.0.0]|ccd01b1 |2023-07-10 |False               |                                                        |                                                                                                                                                        |
+|job-scheduler            |[tags/2.9.0.0]|bf8f0c3 |2023-07-11 |False               |                                                        |                                                                                                                                                        |
+|k-NN                     |[tags/2.9.0.0]|591fff6 |2023-07-13 |True                |opensearch-knn.release-notes-2.9.0.0.md                 |https://raw.githubusercontent.com/opensearch-project/k-NN/2.9.0.0/release-notes/opensearch-knn.release-notes-2.9.0.0.md                                 |
+|ml-commons               |[tags/2.9.0.0]|8f1d47d |2023-07-17 |True                |opensearch-ml-common.release-notes-2.9.0.0.md           |https://raw.githubusercontent.com/opensearch-project/ml-commons/2.9.0.0/release-notes/opensearch-ml-common.release-notes-2.9.0.0.md                     |
+|neural-search            |[tags/2.9.0.0]|f9d64d8 |2023-07-11 |True                |opensearch-neural-search.release-notes-2.9.0.0.md       |https://raw.githubusercontent.com/opensearch-project/neural-search/2.9.0.0/release-notes/opensearch-neural-search.release-notes-2.9.0.0.md              |
+|notifications            |[tags/2.9.0.0]|be24fef |2023-07-11 |True                |opensearch-notifications.release-notes-2.9.0.0.md       |https://raw.githubusercontent.com/opensearch-project/notifications/2.9.0.0/release-notes/opensearch-notifications.release-notes-2.9.0.0.md              |
+|opensearch-observability |[tags/2.9.0.0]|a28655e |2023-07-13 |True                |opensearch-observability.release-notes-2.9.0.0.md       |https://raw.githubusercontent.com/opensearch-project/observability/2.9.0.0/release-notes/opensearch-observability.release-notes-2.9.0.0.md              |
+|opensearch-reports       |[tags/2.9.0.0]|677be51 |2023-07-12 |True                |opensearch-reporting.release-notes-2.9.0.0.md           |https://raw.githubusercontent.com/opensearch-project/reporting/2.9.0.0/release-notes/opensearch-reporting.release-notes-2.9.0.0.md                      |
+|performance-analyzer     |[tags/2.9.0.0]|1f43448 |2023-07-11 |True                |opensearch-performance-analyzer.release-notes-2.9.0.0.md|https://raw.githubusercontent.com/opensearch-project/performance-analyzer/2.9.0.0/release-notes/opensearch-performance-analyzer.release-notes-2.9.0.0.md|
+|security                 |[tags/2.9.0.0]|d548cd2 |2023-07-17 |True                |opensearch-security.release-notes-2.9.0.0.md            |https://raw.githubusercontent.com/opensearch-project/security/2.9.0.0/release-notes/opensearch-security.release-notes-2.9.0.0.md                        |
+|security-analytics       |[tags/2.9.0.0]|629cfae |2023-07-12 |True                |opensearch-security-analytics.release-notes-2.9.0.0.md  |https://raw.githubusercontent.com/opensearch-project/security-analytics/2.9.0.0/release-notes/opensearch-security-analytics.release-notes-2.9.0.0.md    |
+|sql                      |[tags/2.9.0.0]|912f99b |2023-07-12 |True                |opensearch-sql.release-notes-2.9.0.0.md                 |https://raw.githubusercontent.com/opensearch-project/sql/2.9.0.0/release-notes/opensearch-sql.release-notes-2.9.0.0.md                                  |
diff --git a/src/release_notes_automation/release_notes_urls-2.9.0.txt b/src/release_notes_automation/release_notes_urls-2.9.0.txt
@@ -0,0 +1,15 @@
+https://raw.githubusercontent.com/opensearch-project/alerting/2.9.0.0/release-notes/opensearch-alerting.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/anomaly-detection/2.9.0.0/release-notes/opensearch-anomaly-detection.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/asynchronous-search/2.9.0.0/release-notes/opensearch-asynchronous-search.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/common-utils/2.9.0.0/release-notes/opensearch-common-utils.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/geospatial/2.9.0.0/release-notes/opensearch-geospatial.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/k-NN/2.9.0.0/release-notes/opensearch-knn.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/ml-commons/2.9.0.0/release-notes/opensearch-ml-common.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/neural-search/2.9.0.0/release-notes/opensearch-neural-search.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/notifications/2.9.0.0/release-notes/opensearch-notifications.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/observability/2.9.0.0/release-notes/opensearch-observability.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/reporting/2.9.0.0/release-notes/opensearch-reporting.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/performance-analyzer/2.9.0.0/release-notes/opensearch-performance-analyzer.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/security/2.9.0.0/release-notes/opensearch-security.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/security-analytics/2.9.0.0/release-notes/opensearch-security-analytics.release-notes-2.9.0.0.md
+https://raw.githubusercontent.com/opensearch-project/sql/2.9.0.0/release-notes/opensearch-sql.release-notes-2.9.0.0.md
diff --git a/src/run_releasenotes_check.py b/src/run_releasenotes_check.py
@@ -5,6 +5,13 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
+import os
+import requests
+from collections import defaultdict
+import mistune
+import re
+
+from pytablewriter import MarkdownTableWriter
 from manifests.input_manifest import InputManifest
 from release_notes_workflow.release_notes import ReleaseNotes
 from release_notes_workflow.release_notes_check_args import ReleaseNotesCheckArgs
@@ -15,14 +22,170 @@ def main() -> int:
     args = ReleaseNotesCheckArgs()
     console.configure(level=args.logging_level)
     manifest_file = InputManifest.from_file(args.manifest)
-    release_notes = ReleaseNotes(manifest_file, args.date)
-    if args.action == "check":
-        table_output = release_notes.table()
-        table_output.write_table()
+    BUILD_VERSION = manifest_file.build.version
+    # print(f"BUILD_VERSION: {BUILD_VERSION}")
+
+    BASE_FILE_PATH = 'release_notes_automation'
+    table_filename = f'{BASE_FILE_PATH}/release_notes_table-{BUILD_VERSION}.md'
+    urls_filename = f'{BASE_FILE_PATH}/release_notes_urls-{BUILD_VERSION}.txt'
+
+    def format_component_name_from_url(url):
+        start_index = url.find('release-notes/')
+        if start_index == -1:
+            raise ValueError("'release-notes/' not found in the URL")
+        end_index = url.find('.release-notes', start_index)
+        if end_index == -1:
+            raise ValueError("'.release-notes' not found after 'release-notes/'")
+        component_name = url[start_index + len('release-notes/') : end_index]
+        formatted_name = ' '.join(word.capitalize() for word in component_name.split('-'))
+        return formatted_name
+
+    def create_urls_file_if_not_exists():
+        # print("enter create_urls_file function")
+        urls_filepath = os.path.join(os.path.dirname(__file__), urls_filename)
+        if os.path.exists(urls_filepath):
+            # print("URLs file already exists. Skipping creation.")
+            return
+        # print("URLs file does not exist. Creating...")
+
+        release_notes = ReleaseNotes(manifest_file, args.date, args.action)
+        table = release_notes.table()
+
+        table_filepath = os.path.join(os.path.dirname(__file__), table_filename)
+        os.makedirs(os.path.dirname(table_filepath), exist_ok=True)
+        with open(table_filepath, 'w') as table_file:
+            table.dump(table_file)
+
         if args.output is not None:
-            table_output.dump(args.output)
+            shutil.move(table_filepath, args.output)
+        else:
+            with open(table_filepath, 'r') as table_file:
+                print(table_file.read())
+
+        urls = [row[-1].strip() for row in table.value_matrix if row[-1]]
+
+        os.makedirs(os.path.dirname(urls_filepath), exist_ok=True)
+        urls_filepath = os.path.join(os.path.dirname(__file__), urls_filename)
+        with open(urls_filepath, 'w') as urls_file:
+            urls_file.writelines('\n'.join(urls))
+
+    if args.action == "check":
+        # print("check")
+        create_urls_file_if_not_exists()
+
+    elif args.action == "compile":
+        # print("compile")
+        create_urls_file_if_not_exists()
+
+        RELEASENOTES_CATEGORIES = "BREAKING,FEATURES,ENHANCEMENTS,BUG FIXES,INFRASTRUCTURE,DOCUMENTATION,MAINTENANCE,REFACTORING,EXPERIMENTAL"
+        RELEASE_NOTE_MD = f'{BASE_FILE_PATH}/release_notes-{BUILD_VERSION}.md'
+
+        # Clean up URLs in the file
+        urls_filepath = os.path.join(os.path.dirname(__file__), urls_filename)
+        with open(urls_filepath, 'r') as file:
+            urls = [line.strip() for line in file if line.strip()]
+
+        unique_urls = list(set(urls))
+
+        RELEASE_NOTE_MD_path = os.path.join(os.path.dirname(__file__), RELEASE_NOTE_MD)
+        os.makedirs(os.path.dirname(RELEASE_NOTE_MD_path), exist_ok=True)
+        # print(f"RELEASE_NOTE_MD_path: {RELEASE_NOTE_MD_path}")
+
+    # store plugin data
+    plugin_data = defaultdict(lambda: defaultdict(list))
+
+    # # TODO: store unknown categories
+    # unknown_categories = defaultdict(list)
+
+    for url in unique_urls:
+        if not url.startswith("#"):
+            response = requests.get(url)
+            # print(f"Processing URL: {url}")
+
+            if response.status_code == 200:
+                content = response.text
+                plugin_name = format_component_name_from_url(url)
+                # print(f"Plugin Name: {plugin_name}")
+
+                # obtain headings (###) from the content
+                headings = [match.strip() for match in re.findall(r'###.+', content)]
+                if not headings:
+                    continue
+                # print(f"Headings: {headings}")
+
+                # Store content under each heading in respective plugin
+                for i in range(len(headings)):
+                    heading = headings[i].strip()
+
+                    if heading.startswith("### "):
+                        heading = heading[4:]
+                    # print(f"Heading 1: {heading}")
+                    content_start = content.find(headings[i])
+                    if content_start != -1:
+                        if i == len(headings) - 1:
+                            content_to_end = content[content_start:]
+                        else:
+                            content_to_end = content[content_start:content.find(headings[i + 1])]
+                    # remove heading from obtained content to avoid duplication
+                    parts = content_to_end.split('*', 1)
+                    if len(parts) == 2:
+                        content_to_end = '*' + parts[1]
+                    plugin_data[plugin_name][heading].append(content_to_end)
+                    # print("=====================================================")
+                    # print(plugin_data[plugin_name][heading])
+                    # print("=====================================================")
+    # print("Compilation complete.")
+
+    # Markdown renderer
+    markdown = mistune.create_markdown()
+
+    # Filter content for each category
+    with open(RELEASE_NOTE_MD_path, 'w') as outfile:
+        outfile.write(f"Release Notes {BUILD_VERSION}\n\n")
+        for category in RELEASENOTES_CATEGORIES.split(','):
+            outfile.write(markdown(f'\n## {category}\n\n'))
+            for plugin, categories in plugin_data.items():
+                # print(f"Plugin: {plugin}")
+                # print(f"Categories: {categories}")
+                # print("=====================================================")
+
+                if category.lower() in [cat.lower() for cat in categories.keys()]:
+                    # print(f"Category: {category}")
+                    outfile.write(markdown(f'\n### {plugin}\n\n'))
+
+                    for cat, content_list in categories.items():
+                        # print(f"cat: {cat}")
+                        # print(f"content_list: {content_list}")
+                        if cat.lower() == category.lower():
+                            for content in content_list:
+                                # print("=====================================================")
+                                # print(f"Content: {content}")
+                                outfile.write(markdown(content))
+                                outfile.write('\n')
+                    # for content in categories[category]:
+                    #     print("=====================================================")
+                    #     print(f"categories[category]: {categories[category]}")
+                    #     print(f"Content: {content}")
+                    #     outfile.write(markdown(content))
+                    #     outfile.write('\n')
+
+    # print("=====================================================")
+    print(f"Release notes compiled to {RELEASE_NOTE_MD_path}")
+
     return 0
 
+# sub categories obtained:
+# ### Infrastructure
+# ### Enhancements
+# ### Feature
+# ### Maintenance
+# ### Bug Fixes
+# ### Documentation
+# ### Features
+# ### Added
+# ### Experimental Features
+# ### Refactoring
+
 
 if __name__ == "__main__":
     main()