Integtest OSD updated to checkout Ftrepo by default for all OSD plugins

Signed-off-by: Divya Madala <[email protected]>

src/test_workflow/integ_test/integ_test_suite.py

@@ -6,22 +6,15 @@
 # compatible open source license.
 
 import abc
-import json
 import logging
-import os
 from pathlib import Path
 from typing import Any, Dict
 
-from git.git_repository import GitRepository
 from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
-from paths.script_finder import ScriptFinder
-from system.execute import execute
 from test_workflow.dependency_installer import DependencyInstaller
-from test_workflow.integ_test.topology import ClusterEndpoint, NodeEndpoint
 from test_workflow.test_recorder.log_recorder import LogRecorder
 from test_workflow.test_recorder.test_recorder import TestRecorder
-from test_workflow.test_recorder.test_result_data import TestResultData
 from test_workflow.test_result.test_component_results import TestComponentResults
 
 
@@ -33,7 +26,6 @@ class IntegTestSuite(abc.ABC):
     dependency_installer: DependencyInstaller
     bundle_manifest: BundleManifest
     build_manifest: BuildManifest
-    repo: GitRepository
     save_logs: LogRecorder
     additional_cluster_config: dict
 
@@ -61,63 +53,15 @@ def __init__(
         self.bundle_manifest = bundle_manifest
         self.build_manifest = build_manifest
 
-        self.repo = GitRepository(
-            self.component.repository,
-            self.component.commit_id,
-            os.path.join(self.work_dir, self.component.name),
-            test_config.working_directory
-        )
-
         self.save_logs = test_recorder.test_results_logs
         self.additional_cluster_config = None
 
     @abc.abstractmethod
     def execute_tests(self) -> TestComponentResults:
         pass
 
-    def multi_execute_integtest_sh(self, cluster_endpoints: list, security: bool, test_config: str, script: str) -> int:
-
-        def custom_node_endpoint_encoder(node_endpoint: NodeEndpoint) -> dict:
-            return {"endpoint": node_endpoint.endpoint, "port": node_endpoint.port,
-                    "transport": node_endpoint.transport}
-
-        if os.path.exists(script):
-            if len(cluster_endpoints) == 1:
-                single_data_node = cluster_endpoints[0].data_nodes[0]
-                cmd = f"bash {script} -b {single_data_node.endpoint} -p {single_data_node.port} -s {str(security).lower()} -t {self.component.name} -v {self.bundle_manifest.build.version}"
-            else:
-                endpoints_list = []
-                for cluster_details in cluster_endpoints:
-                    endpoints_list.append(cluster_details.__dict__)
-                endpoints_string = json.dumps(endpoints_list, indent=0, default=custom_node_endpoint_encoder)
-                cmd = f"bash {script} -e '"
-                cmd = cmd + endpoints_string + "'"
-                cmd = cmd + f" -s {str(security).lower()} -v {self.bundle_manifest.build.version}"
-            self.repo_work_dir = os.path.join(
-                self.repo.dir,
-                self.test_config.working_directory) if self.test_config.working_directory is not None else self.repo.dir
-            (status, stdout, stderr) = execute(cmd, self.repo_work_dir, True, False)
-            test_result_data = TestResultData(
-                self.component.name,
-                test_config,
-                status,
-                stdout,
-                stderr,
-                self.test_artifact_files
-            )
-            self.save_logs.save_test_result_data(test_result_data)
-            if stderr:
-                logging.info("Stderr reported for component: " + self.component.name)
-                logging.info(stderr)
-            return status
-        else:
-            logging.info(f"{script} does not exist. Skipping integ tests for {self.component.name}")
-            return 0
-
-    def execute_integtest_sh(self, endpoint: str, port: int, security: bool, test_config: str) -> int:
-        cluster_endpoint_port = [ClusterEndpoint("cluster1", [NodeEndpoint(endpoint, port, 9300)], [])]
-        script = ScriptFinder.find_integ_test_script("functionalTestDashboards", "https://github.com/opensearch-project/opensearch-dashboards-functional-test.git")
-        return self.multi_execute_integtest_sh(cluster_endpoint_port, security, test_config, script)
+    def multi_execute_integtest_sh(self, cluster_endpoints: list, security: bool, test_config: str) -> int:
+        pass
 
     def is_security_enabled(self, config: str) -> bool:
         if config in ["with-security", "without-security"]:

src/test_workflow/integ_test/integ_test_suite_opensearch.py

@@ -6,24 +6,29 @@
 # compatible open source license.
 
 import glob
+import json
 import logging
 import os
 from pathlib import Path
 from typing import Any
 
+from git.git_repository import GitRepository
 from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
 from paths.script_finder import ScriptFinder
+from system.execute import execute
 from test_workflow.dependency_installer_opensearch import DependencyInstallerOpenSearch
 from test_workflow.integ_test.integ_test_suite import IntegTestSuite, InvalidTestConfigError
-from test_workflow.integ_test.topology import Topology
+from test_workflow.integ_test.topology import NodeEndpoint, Topology
 from test_workflow.test_recorder.test_recorder import TestRecorder
+from test_workflow.test_recorder.test_result_data import TestResultData
 from test_workflow.test_result.test_component_results import TestComponentResults
 from test_workflow.test_result.test_result import TestResult
 
 
 class IntegTestSuiteOpenSearch(IntegTestSuite):
     dependency_installer: DependencyInstallerOpenSearch
+    repo: GitRepository
 
     def __init__(
         self,
@@ -44,6 +49,12 @@ def __init__(
             bundle_manifest_opensearch,
             build_manifest_opensearch
         )
+        self.repo = GitRepository(
+            self.component.repository,
+            self.component.commit_id,
+            os.path.join(self.work_dir, self.component.name),
+            test_config.working_directory
+        )
 
     def execute_tests(self) -> TestComponentResults:
         test_results = TestComponentResults()
@@ -88,8 +99,44 @@ def __setup_cluster_and_execute_test_config(self, config: str) -> int:
         ) as (endpoints):
             os.chdir(self.work_dir)
             self.pretty_print_message("Running integration tests for " + self.component.name + " " + config)
-            script = ScriptFinder.find_integ_test_script(self.component.name, self.repo.working_directory)
-            return self.multi_execute_integtest_sh(endpoints, security, config, script)
+            return self.multi_execute_integtest_sh(endpoints, security, config)
+
+    def multi_execute_integtest_sh(self, cluster_endpoints: list, security: bool, test_config: str) -> int:
+        script = ScriptFinder.find_integ_test_script(self.component.name, self.repo.working_directory)
+
+        def custom_node_endpoint_encoder(node_endpoint: NodeEndpoint) -> dict:
+            return {"endpoint": node_endpoint.endpoint, "port": node_endpoint.port, "transport": node_endpoint.transport}
+        if os.path.exists(script):
+            if len(cluster_endpoints) == 1:
+                single_data_node = cluster_endpoints[0].data_nodes[0]
+                cmd = f"bash {script} -b {single_data_node.endpoint} -p {single_data_node.port} -s {str(security).lower()} -v {self.bundle_manifest.build.version}"
+            else:
+                endpoints_list = []
+                for cluster_details in cluster_endpoints:
+                    endpoints_list.append(cluster_details.__dict__)
+                endpoints_string = json.dumps(endpoints_list, indent=0, default=custom_node_endpoint_encoder)
+                cmd = f"bash {script} -e '"
+                cmd = cmd + endpoints_string + "'"
+                cmd = cmd + f" -s {str(security).lower()} -v {self.bundle_manifest.build.version}"
+            self.repo_work_dir = os.path.join(
+                self.repo.dir, self.test_config.working_directory) if self.test_config.working_directory is not None else self.repo.dir
+            (status, stdout, stderr) = execute(cmd, self.repo_work_dir, True, False)
+            test_result_data = TestResultData(
+                self.component.name,
+                test_config,
+                status,
+                stdout,
+                stderr,
+                self.test_artifact_files
+            )
+            self.save_logs.save_test_result_data(test_result_data)
+            if stderr:
+                logging.info("Stderr reported for component: " + self.component.name)
+                logging.info(stderr)
+            return status
+        else:
+            logging.info(f"{script} does not exist. Skipping integ tests for {self.component.name}")
+            return 0
 
     @property
     def test_artifact_files(self) -> dict:

src/test_workflow/integ_test/integ_test_suite_opensearch_dashboards.py

@@ -5,17 +5,23 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
+import logging
 import os
 from pathlib import Path
 
+from git.git_repository import GitRepository
 from manifests.build_manifest import BuildManifest
 from manifests.bundle_manifest import BundleManifest
 from manifests.test_manifest import TestComponent
+from paths.script_finder import ScriptFinder
+from system.execute import execute
 from test_workflow.dependency_installer_opensearch import DependencyInstallerOpenSearch
 from test_workflow.dependency_installer_opensearch_dashboards import DependencyInstallerOpenSearchDashboards
 from test_workflow.integ_test.integ_test_suite import IntegTestSuite
 from test_workflow.integ_test.local_test_cluster_opensearch_dashboards import LocalTestClusterOpenSearchDashboards
+from test_workflow.integ_test.topology import ClusterEndpoint, NodeEndpoint
 from test_workflow.test_recorder.test_recorder import TestRecorder
+from test_workflow.test_recorder.test_result_data import TestResultData
 from test_workflow.test_result.test_component_results import TestComponentResults
 from test_workflow.test_result.test_result import TestResult
 
@@ -24,6 +30,7 @@ class IntegTestSuiteOpenSearchDashboards(IntegTestSuite):
     dependency_installer_opensearch_dashboards: DependencyInstallerOpenSearchDashboards
     bundle_manifest_opensearch_dashboards: BundleManifest
     build_manifest_opensearch_dashboards: BuildManifest
+    repo: GitRepository
 
     def __init__(
         self,
@@ -48,6 +55,12 @@ def __init__(
             bundle_manifest_opensearch,
             build_manifest_opensearch
         )
+        self.repo = GitRepository(
+            build_manifest_opensearch_dashboards.components["functionalTestDashboards"].repository,
+            build_manifest_opensearch_dashboards.components["functionalTestDashboards"].commit_id,
+            os.path.join(self.work_dir, self.component.name),
+            test_config.working_directory
+        )
 
         self.dependency_installer_opensearch_dashboards = dependency_installer_opensearch_dashboards
         self.bundle_manifest_opensearch_dashboards = bundle_manifest_opensearch_dashboards
@@ -81,6 +94,38 @@ def __setup_cluster_and_execute_test_config(self, config: str) -> int:
             os.chdir(self.work_dir)
             return self.execute_integtest_sh(endpoint, port, security, config)
 
+    def multi_execute_integtest_sh(self, cluster_endpoints: list, security: bool, test_config: str) -> int:
+        script = ScriptFinder.find_integ_test_script(self.component.name, self.repo.working_directory)
+
+        def custom_node_endpoint_encoder(node_endpoint: NodeEndpoint) -> dict:
+            return {"endpoint": node_endpoint.endpoint, "port": node_endpoint.port, "transport": node_endpoint.transport}
+        if os.path.exists(script):
+            single_data_node = cluster_endpoints[0].data_nodes[0]
+            cmd = f"bash {script} -b {single_data_node.endpoint} -p {single_data_node.port} -s {str(security).lower()} -t {self.component.name} -v {self.bundle_manifest.build.version}"
+            self.repo_work_dir = os.path.join(
+                self.repo.dir, self.test_config.working_directory) if self.test_config.working_directory is not None else self.repo.dir
+            (status, stdout, stderr) = execute(cmd, self.repo_work_dir, True, False)
+            test_result_data = TestResultData(
+                self.component.name,
+                test_config,
+                status,
+                stdout,
+                stderr,
+                self.test_artifact_files
+            )
+            self.save_logs.save_test_result_data(test_result_data)
+            if stderr:
+                logging.info("Stderr reported for component: " + self.component.name)
+                logging.info(stderr)
+            return status
+        else:
+            logging.info(f"{script} does not exist. Skipping integ tests for {self.component.name}")
+            return 0
+
+    def execute_integtest_sh(self, endpoint: str, port: int, security: bool, test_config: str) -> int:
+        cluster_endpoint_port = [ClusterEndpoint("cluster1", [NodeEndpoint(endpoint, port, 9300)], [])]
+        return self.multi_execute_integtest_sh(cluster_endpoint_port, security, test_config)
+
     @property
     def test_artifact_files(self) -> dict:
         return {