Release Notes tracker for component repos

Pipfile

@@ -24,6 +24,10 @@ atomicwrites = "*"
 validators = "*"
 yamlfix = "*"
 yamllint = "*"
+pytablewriter = "*"
+typed-ast = "*"
+zipp = "*"
+importlib-metadata = "*"
 
 [dev-packages]
 

README.md

@@ -17,6 +17,7 @@
     - [Build Numbers](#build-numbers)
     - [Latest Distribution Url](#latest-distribution-url)
     - [Testing the Distribution](#testing-the-distribution)
+    - [Checking Release Notes](#checking-release-notes)
     - [Signing Artifacts](#signing-artifacts)
       - [PGP](#pgp)
       - [Windows](#windows)
@@ -136,6 +137,16 @@ Tests the OpenSearch distribution, including integration, backwards-compatibilit
 
 See [src/test_workflow](./src/test_workflow) for more information.
 
+#### Checking Release Notes
+
+Workflow to check if the release notes exists or not and shows the latest commit for OpenSearch and Dashboard distributions.
+
+To run:
+```bash
+./release_notes.sh check manifests/2.2.0/opensearch-2.2.0.yml --date 2022-07-26
+```
+
+See [src/release_notes_workflow](./src/release_notes_workflow) for more information.
 #### Signing Artifacts
 
 For all types of signing within OpenSearch project we use `opensearch-signer-client` (in progress of being open-sourced) which is a wrapper around internal signing system and is only available for authenticated users. The input requires a path to the build manifest or directory containing all the artifacts or a single artifact. 

release_notes.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+set -e
+
+DIR="$(dirname "$0")"
+"$DIR/run.sh" "$DIR/src/run_releasenotes_check.py" $@

src/git/git_commit.py

@@ -0,0 +1,5 @@
+class GitCommit:
+
+    def __init__(self, id: str, date: str) -> None:
+        self.id = id
+        self.date = date

src/git/git_repository.py

@@ -10,6 +10,7 @@
 from pathlib import Path
 from typing import Any, List
 
+from git.git_commit import GitCommit
 from system.temporary_directory import TemporaryDirectory
 
 
@@ -87,3 +88,14 @@ def path(self, subdirname: str = None) -> Path:
         if subdirname:
             dirname = os.path.join(self.dir, subdirname)
         return Path(dirname)
+
+    def log(self, after: str) -> List[GitCommit]:
+        result = []
+        cmd = f'git log --date=short --after={after} --pretty=format:"%h %ad"'
+        log = self.output(cmd).split("\n")
+        for line in log:
+            if len(line) == 0:
+                continue
+            parts = line.split(" ")
+            result.append(GitCommit(parts[0], parts[1]))
+        return result

src/release_notes_workflow/README.md

@@ -0,0 +1,37 @@
+#### Components Release Notes Check 
+
+Pulls the latest code to check if the release notes exists and whether new commits have been made based on user passed argument `--date`. Outputs a formated markdown table as follows.
+
+*Usage*
+```
+./release_notes.sh check manifests/3.0.0/opensearch-3.0.0.yml --date 2022-07-26
+```
+
+*Sample Output*
+```
+#  OpenSearch CommitID(after 2022-07-26) & Release Notes info
+|          Repo           |   Branch   |CommitID|Commit Date|Release Notes|
+|-------------------------|------------|--------|-----------|-------------|
+|OpenSearch               |tags/2.2.0  |b1017fa |2022-08-08 |True         |
+|common-utils             |tags/2.2.0.0|7d53102 |2022-08-04 |False        |
+|job-scheduler            |tags/2.2.0.0|a501307 |2022-08-02 |True         |
+|ml-commons               |tags/2.2.0.0|a7d2695 |2022-08-08 |True         |
+|performance-analyzer     |tags/2.2.0.0|3a75d7d |2022-08-08 |True         |
+|security                 |tags/2.2.0.0|8e9e583 |2022-08-08 |True         |
+|geospatial               |tags/2.2.0.0|a71475a |2022-08-04 |True         |
+|k-NN                     |tags/2.2.0.0|53185a0 |2022-08-04 |True         |
+|cross-cluster-replication|tags/2.2.0.0|14d871a |2022-08-05 |False        |
+```
+
+The workflow uses the following arguments:
+* `--date`: To check if commit exists after a specific date (in format yyyy-mm-dd, example 2022-07-26).
+* `--output`: To dump the output into an `.md` file, example `--output table.md`).
+
+
+The following options are available.
+
+| name               | description                                                             |
+|--------------------|-------------------------------------------------------------------------|
+| --date             | Shows commit after a specific date.                                     |
+| --output           | Saves the table output to user specified file.  		       	       |
+| -v, --verbose      | Show more verbose output.                                               |

src/release_notes_workflow/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# This page intentionally left blank.

src/release_notes_workflow/release_notes.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+from typing import List
+
+from pytablewriter import MarkdownTableWriter
+
+from git.git_repository import GitRepository
+from manifests.input_manifest import InputComponentFromSource, InputManifest
+from release_notes_workflow.release_notes_component import ReleaseNotesComponents
+from system.temporary_directory import TemporaryDirectory
+
+
+class ReleaseNotes:
+
+    def __init__(self, manifest: InputManifest, date: str) -> None:
+        self.manifest = manifest
+        self.date = date
+
+    def table(self) -> MarkdownTableWriter:
+        table_result = []
+        for component in self.manifest.components.select():
+            if type(component) is InputComponentFromSource:
+                table_result.append(self.check(component))
+        writer = MarkdownTableWriter(
+            table_name=f" {self.manifest.build.name} CommitID(after {self.date}) & Release Notes info",
+            headers=["Repo", "Branch", "CommitID", "Commit Date", "Release Notes"],
+            value_matrix=table_result
+        )
+        return writer
+
+    def check(self, component: InputComponentFromSource) -> List:
+        results = []
+        with TemporaryDirectory(chdir=True) as work_dir:
+            results.append(component.name)
+            results.append(component.ref)
+            with GitRepository(
+                    component.repository,
+                    component.ref,
+                    os.path.join(work_dir.name, component.name),
+                    component.working_directory,
+            ) as repo:
+                logging.debug(f"Checked out {component.name} into {repo.dir}")
+                release_notes = ReleaseNotesComponents.from_component(component, self.manifest.build.version, repo.dir)
+                commits = repo.log(self.date)
+                if len(commits) > 0:
+                    last_commit = commits[-1]
+                    results.append(last_commit.id)
+                    results.append(last_commit.date)
+                else:
+                    results.append(None)
+                    results.append(None)
+                results.append(release_notes.exists())
+        return results

src/release_notes_workflow/release_notes_check_args.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import argparse
+import datetime
+import logging
+from typing import IO
+
+
+class ReleaseNotesCheckArgs:
+    action: str
+    manifest: IO
+    date: str
+    output: str
+
+    def __init__(self) -> None:
+        parser = argparse.ArgumentParser(description="Checkout an OpenSearch Bundle and check for CommitID and Release Notes")
+        parser.add_argument("action", choices=["check"], help="Operation to perform.")
+        parser.add_argument("manifest", type=argparse.FileType("r"), help="Manifest file.")
+        parser.add_argument(
+            "-v",
+            "--verbose",
+            help="Show more verbose output.",
+            action="store_const",
+            default=logging.INFO,
+            const=logging.DEBUG,
+            dest="logging_level",
+        )
+        parser.add_argument(
+            "--date",
+            type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d").date(),
+            dest="date",
+            help="Date to retrieve the commit (in format yyyy-mm-dd, example 2022-07-26)."
+        )
+        parser.add_argument(
+            "--output",
+            help="Output file."
+        )
+        args = parser.parse_args()
+        self.logging_level = args.logging_level
+        self.action = args.action
+        self.manifest = args.manifest
+        self.date = args.date
+        self.output = args.output
+        if self.action == "check" and self.date is None:
+            parser.error("check option requires --date argument")

src/release_notes_workflow/release_notes_component.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import os
+from abc import abstractmethod
+
+from manifests.input_manifest import InputComponentFromSource
+
+
+class ReleaseNotesComponent:
+
+    def __init__(self, component: InputComponentFromSource, build_version: str, root: str) -> None:
+        self.component = component
+        self.build_version = build_version
+        self.root = root
+
+    @property
+    @abstractmethod
+    def filename(self) -> str:
+        pass
+
+    @property
+    def path(self) -> str:
+        return os.path.join(self.root, "release-notes")
+
+    def path_exists(self) -> bool:
+        return os.path.exists(self.path)
+
+    def exists(self) -> bool:
+        return self.path_exists() and any(fname.endswith(self.filename) for fname in os.listdir(self.path))
+
+
+class ReleaseNotesOpenSearch(ReleaseNotesComponent):
+
+    @property
+    def filename(self) -> str:
+        return f'.release-notes-{self.build_version}.md'
+
+
+class ReleaseNotesOpenSearchPlugin(ReleaseNotesComponent):
+
+    @property
+    def filename(self) -> str:
+        return f'.release-notes-{self.build_version}.0.md'
+
+
+class ReleaseNotesComponents:
+
+    @classmethod
+    def from_component(self, component: InputComponentFromSource, build_version: str, root: str) -> ReleaseNotesComponent:
+        if component.name == 'OpenSearch' or component.name == 'OpenSearch-Dashboards':
+            return ReleaseNotesOpenSearch(component, build_version, root)
+        else:
+            return ReleaseNotesOpenSearchPlugin(component, build_version, root)

src/run_releasenotes_check.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+from manifests.input_manifest import InputManifest
+from release_notes_workflow.release_notes import ReleaseNotes
+from release_notes_workflow.release_notes_check_args import ReleaseNotesCheckArgs
+from system import console
+
+
+def main() -> int:
+    args = ReleaseNotesCheckArgs()
+    console.configure(level=args.logging_level)
+    manifest_file = InputManifest.from_file(args.manifest)
+    release_notes = ReleaseNotes(manifest_file, args.date)
+    if args.action == "check":
+        table_output = release_notes.table()
+        table_output.write_table()
+        if args.output is not None:
+            table_output.dump(args.output)
+    return 0
+
+
+if __name__ == "__main__":
+    main()

tests/test_run_releasenotes_check.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import datetime
+import os
+import unittest
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+
+from release_notes_workflow.release_notes_check_args import ReleaseNotesCheckArgs
+from run_releasenotes_check import main
+
+
+class TestRunReleaseNotesCheck(unittest.TestCase):
+
+    OPENSEARCH_MANIFEST = os.path.realpath(
+        os.path.join(
+            os.path.dirname(__file__),
+            "..",
+            "manifests",
+            "templates",
+            "opensearch",
+            "2.x",
+            "manifest.yml"
+        )
+    )
+
+    @pytest.fixture(autouse=True)
+    def _capfd(self, capfd: Any) -> None:
+        self.capfd = capfd
+
+    @patch("argparse._sys.argv", ["run_releasenotes_check.py", "--help"])
+    def test_usage(self) -> None:
+        with self.assertRaises(SystemExit):
+            main()
+
+        out, _ = self.capfd.readouterr()
+        self.assertTrue(out.startswith("usage:"))
+
+    @patch("argparse._sys.argv", ["run_releasenotes_check.py", "check", OPENSEARCH_MANIFEST, "--date", "2022-07-26"])
+    @patch('subprocess.check_call')
+    @patch("run_releasenotes_check.ReleaseNotes")
+    @patch("run_releasenotes_check.main", return_value=0)
+    def test_main(self, *mocks: Any) -> None:
+        self.assertEqual(ReleaseNotesCheckArgs().action, 'check')
+        self.assertEqual(ReleaseNotesCheckArgs().date, datetime.date(2022, 7, 26))
+        self.assertTrue(ReleaseNotesCheckArgs().manifest)
+
+    @patch('subprocess.check_call')
+    @patch("argparse._sys.argv", ["run_releasenotes_check.py", "check", OPENSEARCH_MANIFEST, "--date", "2022-07-26", "--output", "test.md"])
+    @patch("run_releasenotes_check.ReleaseNotes")
+    @patch("run_releasenotes_check.main", return_value=0)
+    def test_main_with_save(self, *mocks: Any) -> None:
+        self.assertEqual(ReleaseNotesCheckArgs().output, "test.md")