Add rpm support in integTest framework (2nd PR) (#2000)

* Add rpm support in integTest framework (2nd PR)

Signed-off-by: Peter Zhu <[email protected]>

* Add syntax checks

Signed-off-by: Peter Zhu <[email protected]>

* Update tests

Signed-off-by: Peter Zhu <[email protected]>

* Fix isort

Signed-off-by: Peter Zhu <[email protected]>

* More typo fixes

Signed-off-by: Peter Zhu <[email protected]>

* Rename vars

Signed-off-by: Peter Zhu <[email protected]>

* More map removal

Signed-off-by: Peter Zhu <[email protected]>

* More tweaks

Signed-off-by: Peter Zhu <[email protected]>

src/test_workflow/integ_test/distribution.py

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+from abc import ABC, abstractmethod
+
+
+class Distribution(ABC):
+    def __enter__(self) -> 'Distribution':
+        return self
+
+    def __init__(self, filename: str, version: str, work_dir: str) -> None:
+        self.filename = filename
+        self.version = version
+        self.work_dir = work_dir
+
+    @property
+    @abstractmethod
+    def install_dir(self) -> str:
+        """
+        Return the install directory for the distribution
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def config_dir(self, bundle_name: str) -> str:
+        """
+        Return the config directory for the distribution
+        """
+        pass
+
+    @abstractmethod
+    def install(self) -> None:
+        """
+        The detailed method to install the distribution before start the service
+        """
+        pass
+
+    @property
+    @abstractmethod
+    def start_cmd(self) -> str:
+        """
+        Return the start command for the distribution
+        """
+        pass
+
+    def uninstall(self) -> None:
+        """
+        Allow distribution that is not 'tar' to do proper cleanup
+        """
+        pass

src/test_workflow/integ_test/distribution_rpm.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+import subprocess
+
+from test_workflow.integ_test.distribution import Distribution
+
+
+class DistributionRpm(Distribution):
+    def __init__(self, filename: str, version: str, work_dir: str) -> None:
+        super().__init__(filename, version, work_dir)
+
+    @property
+    def install_dir(self) -> str:
+        return os.path.join(os.sep, "usr", "share", self.filename)
+
+    @property
+    def config_dir(self) -> str:
+        return os.path.join(os.sep, "etc", self.filename)
+
+    def install(self, bundle_name: str) -> None:
+        logging.info(f"Installing {bundle_name} in {self.install_dir}")
+        logging.info("rpm installation requires sudo, script will exit if current user does not have sudo access")
+        rpm_install_cmd = " ".join(
+            [
+                'yum',
+                'remove',
+                '-y',
+                self.filename,
+                '&&',
+                'yum',
+                'install',
+                '-y',
+                bundle_name
+            ]
+        )
+        subprocess.check_call(rpm_install_cmd, cwd=self.work_dir, shell=True)
+
+    @property
+    def start_cmd(self) -> str:
+        return f"systemctl start {self.filename}"
+
+    def uninstall(self) -> None:
+        logging.info("Uninstall {self.filename} package after the test")
+        subprocess.check_call(f"yum remove -y {self.filename}", shell=True)

src/test_workflow/integ_test/distribution_tar.py

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+import tarfile
+
+from test_workflow.integ_test.distribution import Distribution
+
+
+class DistributionTar(Distribution):
+    def __init__(self, filename: str, version: str, work_dir: str) -> None:
+        super().__init__(filename, version, work_dir)
+
+    @property
+    def install_dir(self) -> str:
+        return os.path.join(self.work_dir, f"{self.filename}-{self.version}")
+
+    @property
+    def config_dir(self) -> str:
+        return os.path.join(self.install_dir, "config")
+
+    def install(self, bundle_name: str) -> None:
+        logging.info(f"Installing {bundle_name} in {self.install_dir}")
+        with tarfile.open(bundle_name, 'r:gz') as bundle_tar:
+            bundle_tar.extractall(self.work_dir)
+
+    @property
+    def start_cmd(self) -> str:
+        start_cmd_map = {
+            "opensearch": "./opensearch-tar-install.sh",
+            "opensearch-dashboards": "./opensearch-dashboards",
+        }
+        return start_cmd_map[self.filename]

src/test_workflow/integ_test/distributions.py

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+
+from test_workflow.integ_test.distribution import Distribution
+from test_workflow.integ_test.distribution_rpm import DistributionRpm
+from test_workflow.integ_test.distribution_tar import DistributionTar
+
+
+class Distributions:
+    DISTRIBUTIONS_MAP = {
+        "tar": DistributionTar,
+        "rpm": DistributionRpm,
+    }
+
+    @classmethod
+    def from_name(cls, name: str) -> Distribution:
+        klass = cls.DISTRIBUTIONS_MAP.get(name, None)
+        if not klass:
+            raise ValueError(f"Unsupported distribution: {name}")
+        return klass  # type: ignore[return-value]
+
+    @classmethod
+    def get_distribution(cls, filename: str, distribution: str, version: str, work_dir: str) -> Distribution:
+        klass = cls.from_name(distribution)
+        logging.info(f"{filename} distribution: {distribution}")
+        return klass(filename, version, work_dir)  # type: ignore[no-any-return, operator]

src/test_workflow/integ_test/local_test_cluster.py

@@ -40,6 +40,7 @@ def __init__(
 
         self.service_opensearch = ServiceOpenSearch(
             self.manifest.build.version,
+            self.manifest.build.distribution,
             self.additional_cluster_config,
             self.security_enabled,
             self.dependency_installer,

src/test_workflow/integ_test/local_test_cluster_opensearch_dashboards.py

@@ -45,15 +45,15 @@ def __init__(
 
         self.service_opensearch = ServiceOpenSearch(
             self.manifest_opensearch.build.version,
+            self.manifest_opensearch.build.distribution,
             {},
             self.security_enabled,
             self.dependency_installer_opensearch,
             self.work_dir)
 
-        build = self.manifest_opensearch_dashboards.build
-
         self.service_opensearch_dashboards = ServiceOpenSearchDashboards(
-            build.version,
+            self.manifest_opensearch_dashboards.build.version,
+            self.manifest_opensearch_dashboards.build.distribution,
             self.additional_cluster_config,
             self.security_enabled,
             self.dependency_installer_opensearch_dashboards,

src/test_workflow/integ_test/service.py

@@ -6,6 +6,7 @@
 
 import abc
 import logging
+import os
 import time
 
 import requests
@@ -19,9 +20,10 @@ class Service(abc.ABC):
     Abstract base class for all types of test clusters.
     """
 
-    def __init__(self, work_dir, version, security_enabled, additional_config, dependency_installer):
+    def __init__(self, work_dir, version, distribution, security_enabled, additional_config, dependency_installer):
         self.work_dir = work_dir
         self.version = version
+        self.distribution = distribution
         self.security_enabled = security_enabled
         self.additional_config = additional_config
         self.dependency_installer = dependency_installer
@@ -43,6 +45,8 @@ def terminate(self):
 
         self.return_code = self.process_handler.terminate()
 
+        self.uninstall()
+
         return ServiceTerminationResult(self.return_code, self.process_handler.stdout_data, self.process_handler.stderr_data, self.log_files)
 
     def endpoint(self):
@@ -80,6 +84,12 @@ def service_alive(self):
         else:
             return False
 
+    def download(self):
+        logging.info("Downloading bundle artifact")
+        bundle_name = self.dependency_installer.download_dist(self.work_dir)
+        logging.info(f"Downloaded bundle to {os.path.realpath(bundle_name)}")
+        return bundle_name
+
     def wait_for_service(self):
         logging.info("Waiting for service to become available")
 

src/test_workflow/integ_test/service_opensearch.py

@@ -6,33 +6,34 @@
 
 import logging
 import os
-import tarfile
 
 import requests
 import yaml
 
+from test_workflow.integ_test.distributions import Distributions
 from test_workflow.integ_test.service import Service
 
 
 class ServiceOpenSearch(Service):
     def __init__(
         self,
         version,
+        distribution,
         additional_config,
         security_enabled,
         dependency_installer,
         work_dir
     ):
-        super().__init__(work_dir, version, security_enabled, additional_config, dependency_installer)
+        super().__init__(work_dir, version, distribution, security_enabled, additional_config, dependency_installer)
 
+        self.dist = Distributions.get_distribution("opensearch", distribution, version, work_dir)
         self.dependency_installer = dependency_installer
-
-        self.install_dir = os.path.join(self.work_dir, f"opensearch-{self.version}")
+        self.install_dir = self.dist.install_dir
 
     def start(self):
-        self.__download()
+        self.dist.install(self.download())
 
-        self.opensearch_yml_dir = os.path.join(self.install_dir, "config", "opensearch.yml")
+        self.opensearch_yml_dir = os.path.join(self.dist.config_dir, "opensearch.yml")
         self.security_plugin_dir = os.path.join(self.install_dir, "plugins", "opensearch-security")
 
         if not self.security_enabled and os.path.isdir(self.security_plugin_dir):
@@ -41,20 +42,11 @@ def start(self):
         if self.additional_config:
             self.__add_plugin_specific_config(self.additional_config)
 
-        self.process_handler.start("./opensearch-tar-install.sh", self.install_dir)
+        self.process_handler.start(self.dist.start_cmd, self.install_dir)
         logging.info(f"Started OpenSearch with parent PID {self.process_handler.pid}")
 
-    def __download(self):
-        logging.info(f"Creating local test cluster in {self.work_dir}")
-        logging.info("Downloading bundle")
-        bundle_name = self.dependency_installer.download_dist(self.work_dir)
-        logging.info(f"Downloaded bundle to {os.path.realpath(bundle_name)}")
-
-        logging.info(f"Unpacking {bundle_name}")
-        with tarfile.open(bundle_name, 'r') as bundle_tar:
-            bundle_tar.extractall(self.work_dir)
-
-        logging.info(f"Unpacked {bundle_name}")
+    def uninstall(self):
+        self.dist.uninstall()
 
     def url(self, path=""):
         return f'{"https" if self.security_enabled else "http"}://{self.endpoint()}:{self.port()}{path}'

src/test_workflow/integ_test/service_opensearch_dashboards.py

@@ -7,31 +7,32 @@
 import logging
 import os
 import subprocess
-import tarfile
 
 import requests
 import yaml
 
+from test_workflow.integ_test.distributions import Distributions
 from test_workflow.integ_test.service import Service
 
 
 class ServiceOpenSearchDashboards(Service):
     def __init__(
         self,
         version,
+        distribution,
         additional_config,
         security_enabled,
         dependency_installer,
         work_dir
     ):
-        super().__init__(work_dir, version, security_enabled, additional_config, dependency_installer)
-        self.install_dir = os.path.join(self.work_dir, f"opensearch-dashboards-{self.version}")
+        super().__init__(work_dir, version, distribution, security_enabled, additional_config, dependency_installer)
+        self.dist = Distributions.get_distribution("opensearch-dashboards", distribution, version, work_dir)
+        self.install_dir = self.dist.install_dir
 
     def start(self):
-        logging.info(f"Starting OpenSearch Dashboards service from {self.work_dir}")
-        self.__download()
+        self.dist.install(self.download())
 
-        self.opensearch_dashboards_yml_dir = os.path.join(self.install_dir, "config", "opensearch_dashboards.yml")
+        self.opensearch_dashboards_yml_dir = os.path.join(self.dist.config_dir, "opensearch_dashboards.yml")
         self.executable_dir = os.path.join(self.install_dir, "bin")
 
         if not self.security_enabled:
@@ -42,9 +43,12 @@ def start(self):
         if self.additional_config:
             self.__add_plugin_specific_config(self.additional_config)
 
-        self.process_handler.start("./opensearch-dashboards", self.executable_dir)
+        self.process_handler.start(self.dist.start_cmd, self.executable_dir)
         logging.info(f"Started OpenSearch Dashboards with parent PID {self.process_handler.pid}")
 
+    def uninstall(self):
+        self.dist.uninstall()
+
     def __set_logging_dest(self):
         self.log_dir = os.path.join(self.install_dir, "logs")
         os.makedirs(self.log_dir, exist_ok=True)
@@ -53,22 +57,11 @@ def __set_logging_dest(self):
     def __remove_security(self):
         self.security_plugin_dir = os.path.join(self.install_dir, "plugins", "securityDashboards")
         if os.path.isdir(self.security_plugin_dir):
-            subprocess.check_call("./opensearch-dashboards-plugin remove securityDashboards", cwd=self.executable_dir, shell=True)
+            subprocess.check_call("./opensearch-dashboards-plugin remove --allow-root securityDashboards", cwd=self.executable_dir, shell=True)
 
         with open(self.opensearch_dashboards_yml_dir, "w") as yamlfile:
             yamlfile.close()
 
-    def __download(self):
-        logging.info("Downloading OpenSearch Dashboards bundle")
-        bundle_name = self.dependency_installer.download_dist(self.work_dir)
-        logging.info(f"Downloaded bundle to {os.path.realpath(bundle_name)}")
-
-        logging.info(f"Unpacking {bundle_name}")
-        with tarfile.open(bundle_name, 'r') as bundle_tar:
-            bundle_tar.extractall(self.work_dir)
-
-        logging.info(f"Unpacked {bundle_name}")
-
     def url(self, path=""):
         return f'http://{self.endpoint()}:{self.port()}{path}'