Add JAVA_HOME arguments in manifest workflow from template manifests. (…

…opensearch-project#2358) * Adding java home arg Signed-off-by: Zelin Hao <[email protected]> * Attach tests Signed-off-by: Zelin Hao <[email protected]> * Change to get ci data from templates instead of hardcoding Signed-off-by: Zelin Hao <[email protected]> * Fix flake8 error Signed-off-by: Zelin Hao <[email protected]> * Keep simplifying the workflow. Signed-off-by: Zelin Hao <[email protected]> * More changes Signed-off-by: Zelin Hao <[email protected]>
dblock · Jul 27, 2022 · cbf57ab · cbf57ab
1 parent 2c14ef5
commit cbf57ab
Show file tree

Hide file tree

Showing 13 changed files with 185 additions and 39 deletions.
diff --git a/.github/workflows/manifests.yml b/.github/workflows/manifests.yml
@@ -5,7 +5,8 @@ on:
   push:
   pull_request:
     paths:
-      - manifests/**/*.yml
+      - 'manifests/**/*.yml'
+      - '!manifests/templates/**/'
   schedule:
     - cron: 0 0 * * *
 

diff --git a/manifests/templates/opensearch-dashboards/1.x/manifest.yml b/manifests/templates/opensearch-dashboards/1.x/manifest.yml
@@ -0,0 +1,12 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch Dashboards
+  version: 'replace'
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2
+components:
+  - name: OpenSearch-Dashboards
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    ref: main
diff --git a/manifests/templates/opensearch-dashboards/2.x/manifest.yml b/manifests/templates/opensearch-dashboards/2.x/manifest.yml
@@ -0,0 +1,14 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch Dashboards
+  version: 'replace'
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2
+components:
+  - name: OpenSearch-Dashboards
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    ref: main
+    checks:
+      - npm:package:version
diff --git a/manifests/templates/opensearch-dashboards/3.x/manifest.yml b/manifests/templates/opensearch-dashboards/3.x/manifest.yml
@@ -0,0 +1,12 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch Dashboards
+  version: 'replace'
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2
+components:
+  - name: OpenSearch-Dashboards
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    ref: main
diff --git a/manifests/templates/opensearch-dashboards/default/manifest.yml b/manifests/templates/opensearch-dashboards/default/manifest.yml
@@ -0,0 +1,14 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch Dashboards
+  version: 'replace'
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2
+components:
+  - name: OpenSearch-Dashboards
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    ref: main
+    checks:
+      - npm:package:version
diff --git a/manifests/templates/opensearch/1.x/manifest.yml b/manifests/templates/opensearch/1.x/manifest.yml
@@ -0,0 +1,16 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch
+  version: 'replace'
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-11
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: main
+    checks:
+      - gradle:publish
+      - gradle:properties:version
diff --git a/manifests/templates/opensearch/2.x/manifest.yml b/manifests/templates/opensearch/2.x/manifest.yml
@@ -0,0 +1,16 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch
+  version: 'replace'
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-17
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: main
+    checks:
+      - gradle:publish
+      - gradle:properties:version
diff --git a/manifests/templates/opensearch/3.x/manifest.yml b/manifests/templates/opensearch/3.x/manifest.yml
@@ -0,0 +1,16 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch
+  version: 'replace'
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-17
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: main
+    checks:
+      - gradle:publish
+      - gradle:properties:version
diff --git a/manifests/templates/opensearch/default/manifest.yml b/manifests/templates/opensearch/default/manifest.yml
@@ -0,0 +1,16 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch
+  version: 'replace'
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-17
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: main
+    checks:
+      - gradle:publish
+      - gradle:properties:version
diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py
@@ -11,7 +11,7 @@
 from abc import abstractmethod
 from typing import Dict, List, Type, Union
 
-from manifests.input_manifest import InputManifest
+from manifests.input_manifest import InputComponents, InputManifest
 from manifests.manifests import Manifests
 from manifests_workflow.component_opensearch import ComponentOpenSearch
 from manifests_workflow.component_opensearch_dashboards_min import ComponentOpenSearchDashboardsMin
@@ -48,7 +48,12 @@ def files(self, name: str) -> List:
         return results
 
     @abstractmethod
-    def update(self, min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOpenSearchDashboardsMin]], component_klass: Type[ComponentOpenSearch], keep: bool = False) -> None:
+    def update(
+        self,
+        min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOpenSearchDashboardsMin]],
+        component_klass: Type[ComponentOpenSearch],
+        keep: bool = False,
+    ) -> None:
         known_versions = self.versions
         logging.info(f"Known versions: {known_versions}")
         main_versions: Dict = {}
@@ -106,30 +111,23 @@ def update(self, min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOp
                 self.add_to_cron(release_version)
 
     def create_manifest(self, version: str, components: List = []) -> InputManifest:
-        image_map = {
-            "opensearch": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2",
-            "opensearch-dashboards": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2"
-        }
-
-        data: Dict = {
-            "schema-version": "1.0",
-            "build": {
-                "name": self.name,
-                "version": version
-            },
-            "ci": {
-                "image": {
-                    "name": image_map[self.prefix]
-                }
-            },
-            "components": [],
-        }
+        templates_base_path = os.path.join(self.manifests_path(), "templates")
+        template_version_folder = version.split(".")[0] + ".x"
+        template_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest.yml")
+        if not os.path.exists(template_full_path):
+            template_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest.yml")
+
+        manifest = InputManifest.from_file(open(template_full_path))
+
+        manifest.build.version = version
+        manifests_components = []
 
         for component in components:
             logging.info(f" Adding {component.name}")
-            data["components"].append(component.to_dict())
+            manifests_components.append(component.to_dict())
 
-        return InputManifest(data)
+        manifest.components = InputComponents(manifests_components)  # type: ignore
+        return manifest
 
     def write_manifest(self, version: str, components: List = []) -> None:
         logging.info(f"Creating new version: {version}")
@@ -151,10 +149,7 @@ def add_to_cron(self, version: str) -> None:
         if cron_entry in data:
             raise ValueError(f"{jenkinsfile} already contains an entry for {self.prefix} {version}")
 
-        data = data.replace(
-            "parameterizedCron '''\n",
-            f"parameterizedCron '''\n{' ' * 12}{cron_entry}"
-        )
+        data = data.replace("parameterizedCron '''\n", f"parameterizedCron '''\n{' ' * 12}{cron_entry}")
 
         with open(jenkinsfile, "w") as f:
             f.write(data)

diff --git a/tests/tests_manifests_workflow/test_input_manifests.py b/tests/tests_manifests_workflow/test_input_manifests.py
@@ -17,26 +17,52 @@ def test_manifests_path(self) -> None:
         self.assertEqual(path, InputManifests.manifests_path())
 
     def test_create_manifest_opensearch(self) -> None:
-        input_manifests = InputManifests("opensearch")
+        input_manifests = InputManifests("OpenSearch")
         input_manifest = input_manifests.create_manifest("1.2.3", [])
         self.assertEqual(
             input_manifest.to_dict(),
             {
                 "schema-version": "1.0",
-                "build": {"name": "opensearch", "version": "1.2.3"},
-                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2"}},
+                "build": {"name": "OpenSearch", "version": "1.2.3"},
+                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2",
+                                 "args": "-e JAVA_HOME=/opt/java/openjdk-11"}},
+            },
+        )
+
+    def test_create_manifest_opensearch_from_default(self) -> None:
+        input_manifests = InputManifests("OpenSearch")
+        input_manifest = input_manifests.create_manifest("0.2.3", [])
+        self.assertEqual(
+            input_manifest.to_dict(),
+            {
+                "schema-version": "1.0",
+                "build": {"name": "OpenSearch", "version": "0.2.3"},
+                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2",
+                                 "args": "-e JAVA_HOME=/opt/java/openjdk-17"}},
             },
         )
 
     def test_create_manifest_opensearch_dashboards(self) -> None:
-        input_manifests = InputManifests("opensearch-dashboards")
+        input_manifests = InputManifests("OpenSearch Dashboards")
         input_manifest = input_manifests.create_manifest("1.2.3", [])
         self.assertEqual(
             input_manifest.to_dict(),
             {
                 "schema-version": "1.0",
-                "build": {"name": "opensearch-dashboards", "version": "1.2.3"},
-                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2"}},
+                "build": {"name": "OpenSearch Dashboards", "version": "1.2.3"},
+                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2", }},
+            },
+        )
+
+    def test_create_manifest_opensearch_dashboards_from_default(self) -> None:
+        input_manifests = InputManifests("OpenSearch Dashboards")
+        input_manifest = input_manifests.create_manifest("4.2.3", [])
+        self.assertEqual(
+            input_manifest.to_dict(),
+            {
+                "schema-version": "1.0",
+                "build": {"name": "OpenSearch Dashboards", "version": "4.2.3"},
+                "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2", }},
             },
         )
 

diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py
@@ -30,14 +30,17 @@ def test_files(self) -> None:
 
     @patch("os.makedirs")
     @patch("os.chdir")
+    @patch("manifests_workflow.input_manifests.InputComponents")
+    @patch("manifests_workflow.input_manifests.InputManifest.from_file")
     @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron")
     @patch("manifests_workflow.input_manifests.InputManifest.from_path")
     @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearchMin")
     @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearch")
     @patch("manifests_workflow.input_manifests.InputManifest")
     def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: MagicMock,
                     mock_component_opensearch_min: MagicMock, mock_input_manifest_from_path: MagicMock,
-                    mock_add_to_cron: MagicMock, *mocks: MagicMock) -> None:
+                    mock_add_to_cron: MagicMock, mock_input_manifest_from_file: MagicMock,
+                    mock_input_manifest_component: MagicMock, *mocks: MagicMock) -> None:
         mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch")
         mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"]
         mock_component_opensearch_min.checkout.return_value = MagicMock(version="0.9.0")
@@ -48,7 +51,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch:
         }
         manifests = InputManifestsOpenSearch()
         manifests.update()
-        self.assertEqual(mock_input_manifest().to_file.call_count, 2)
+        self.assertEqual(mock_input_manifest_from_file().to_file.call_count, 2)
         calls = [
             call(
                 os.path.join(
@@ -65,7 +68,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch:
                 )
             ),
         ]
-        mock_input_manifest().to_file.assert_has_calls(calls)
+        mock_input_manifest_from_file().to_file.assert_has_calls(calls)
         mock_add_to_cron.assert_has_calls([
             call('0.10.0'),
             call('0.9.0')

diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py
@@ -29,19 +29,24 @@ def test_files(self) -> None:
 
     @patch("os.makedirs")
     @patch("os.chdir")
+    @patch("manifests_workflow.input_manifests.InputComponents")
+    @patch("manifests_workflow.input_manifests.InputManifest.from_file")
     @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron")
     @patch("manifests_workflow.input_manifests.InputManifest.from_path")
     @patch("manifests_workflow.input_manifests_opensearch_dashboards.ComponentOpenSearchDashboardsMin")
     @patch("manifests_workflow.input_manifests.InputManifest")
     def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_min: MagicMock,
-                    mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock, *mocks: MagicMock) -> None:
+                    mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock,
+                    mock_input_manifest_from_file: MagicMock, mock_input_manifest_component: MagicMock,
+                    *mocks: MagicMock) -> None:
         mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch-Dashboards")
         mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"]
         mock_component_opensearch_min.checkout.return_value = MagicMock(version="0.9.0")
         mock_input_manifest_from_path.return_value = MagicMock(components=[])
+
         manifests = InputManifestsOpenSearchDashboards()
         manifests.update()
-        self.assertEqual(mock_input_manifest().to_file.call_count, 1)
+        self.assertEqual(mock_input_manifest_from_file().to_file.call_count, 1)
         calls = [
             call(
                 os.path.join(
@@ -51,7 +56,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_
                 )
             )
         ]
-        mock_input_manifest().to_file.assert_has_calls(calls)
+        mock_input_manifest_from_file().to_file.assert_has_calls(calls)
         mock_add_to_cron.assert_has_calls([
             call('0.9.0')
         ])