From b0d2307a1ee90cefbdc505dc363dbc60ec9117b9 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 19 Apr 2022 23:24:45 +0000 Subject: [PATCH 1/8] Add rpm support in integTest framework (2nd PR) Signed-off-by: Peter Zhu --- src/test_workflow/integ_test/distribution.py | 63 +++++++++++++++++++ .../integ_test/distribution_rpm.py | 57 +++++++++++++++++ .../integ_test/distribution_tar.py | 40 ++++++++++++ src/test_workflow/integ_test/distributions.py | 31 +++++++++ .../integ_test/local_test_cluster.py | 1 + ...ocal_test_cluster_opensearch_dashboards.py | 6 +- src/test_workflow/integ_test/service.py | 12 +++- .../integ_test/service_opensearch.py | 27 +++----- .../service_opensearch_dashboards.py | 32 ++++------ 9 files changed, 229 insertions(+), 40 deletions(-) create mode 100644 src/test_workflow/integ_test/distribution.py create mode 100644 src/test_workflow/integ_test/distribution_rpm.py create mode 100644 src/test_workflow/integ_test/distribution_tar.py create mode 100644 src/test_workflow/integ_test/distributions.py diff --git a/src/test_workflow/integ_test/distribution.py b/src/test_workflow/integ_test/distribution.py new file mode 100644 index 0000000000..7829e033aa --- /dev/null +++ b/src/test_workflow/integ_test/distribution.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import errno +import logging +import os +import shutil +import subprocess +from abc import ABC, abstractmethod +from typing import Any, List + +from system.process import Process + +class Distribution(ABC): + def __enter__(self) -> 'Distribution': + return self + + def __init__(self, filename: str, distribution: str, version: str, work_dir: str) -> None: + self.filename = filename + self.distribution = distribution + self.version = version + self.work_dir = work_dir + + @property + @abstractmethod + def get_install_dir(self) -> str: + """ + Return the install directory for the distribution + """ + pass + + @property + @abstractmethod + def get_config_dir(self, bundle_name: str) -> str: + """ + Return the config directory for the distribution + """ + pass + + @abstractmethod + def install_distribution(self) ->None: + """ + The detailed method to install the distribution before start the service + """ + pass + + @property + @abstractmethod + def get_start_cmd(self) -> str: + """ + Return the start command for the distribution + """ + pass + + def cleanup(self) -> None: + """ + Allow distribution that is not 'tar' to do proper cleanup + """ + pass + diff --git a/src/test_workflow/integ_test/distribution_rpm.py b/src/test_workflow/integ_test/distribution_rpm.py new file mode 100644 index 0000000000..a6987e607d --- /dev/null +++ b/src/test_workflow/integ_test/distribution_rpm.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import errno +import logging +import os +import tarfile +import shutil +import subprocess + +from system.process import Process +from test_workflow.integ_test.distribution import Distribution + +class DistributionRpm(Distribution): + def __init__(self, filename: str, distribution: str, version: str, work_dir: str) -> None: + super().__init__(filename, distribution, version, work_dir) + + @property + def get_install_dir(self) -> str: + return os.path.join(os.sep, "usr", "share", self.filename) + + @property + def get_config_dir(self) -> str: + return os.path.join(os.sep, "etc", self.filename) + + def install_distribution(self, bundle_name: str) ->None: + logging.info(f"Installing {bundle_name} in {self.get_install_dir}") + logging.info("rpm installation requires sudo, script will exit if current user does not have sudo access") + rpm_install_cmd = " ".join( + [ + 'yum', + 'remove', + '-y', + self.filename, + '&&', + 'yum', + 'install', + '-y', + bundle_name + ] + ) + subprocess.check_call(rpm_install_cmd, cwd=self.work_dir, shell=True) + + @property + def get_start_cmd(self) -> str: + start_cmd_map = { + "opensearch": "systemctl start opensearch", + "opensearch-dashboards": "systemctl start opensearch-dashboards", + } + return start_cmd_map[self.filename] + + def cleanup(self) -> None: + logging.info("Clean up packages after the test") + subprocess.check_call(f"yum remove -y '{self.filename}*'", shell=True) diff --git a/src/test_workflow/integ_test/distribution_tar.py b/src/test_workflow/integ_test/distribution_tar.py new file mode 100644 index 0000000000..39a26f5f45 --- /dev/null +++ b/src/test_workflow/integ_test/distribution_tar.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import errno +import logging +import os +import tarfile +import shutil +import subprocess + +from test_workflow.integ_test.distribution import Distribution + +class DistributionTar(Distribution): + def __init__(self, filename: str, distribution: str, version: str, work_dir: str) -> None: + super().__init__(filename, distribution, version, work_dir) + + @property + def get_install_dir(self) -> str: + return os.path.join(self.work_dir, f"{self.filename}-{self.version}") + + @property + def get_config_dir(self) -> str: + return os.path.join(self.get_install_dir, "config") + + def install_distribution(self, bundle_name: str) ->None: + logging.info(f"Installing {bundle_name} in {self.get_install_dir}") + with tarfile.open(bundle_name, 'r') as bundle_tar: + bundle_tar.extractall(self.work_dir) + + @property + def get_start_cmd(self) -> str: + start_cmd_map = { + "opensearch": "./opensearch-tar-install.sh", + "opensearch-dashboards": "./opensearch-dashboards", + } + return start_cmd_map[self.filename] + diff --git a/src/test_workflow/integ_test/distributions.py b/src/test_workflow/integ_test/distributions.py new file mode 100644 index 0000000000..a47b90008f --- /dev/null +++ b/src/test_workflow/integ_test/distributions.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import logging + +from test_workflow.integ_test.distribution import Distribution +from test_workflow.integ_test.distribution_tar import DistributionTar +from test_workflow.integ_test.distribution_rpm import DistributionRpm + + +class Distributions: + DISTRIBUTION_MAP = { + "tar": DistributionTar, + "rpm": DistributionRpm, + } + + @classmethod + def from_name(cls, name: str) -> Distribution: + klass = cls.DISTRIBUTION_MAP.get(name, None) + if not klass: + raise ValueError(f"Unsupported distribution: {name}") + return klass # type: ignore[return-value] + + @classmethod + def get_distribution(cls, filename: str, distribution: str, version: str, work_dir: str) -> Distribution: + klass = cls.from_name(distribution) + logging.info(f"{filename} distribution: {distribution}") + return klass(filename, distribution, version, work_dir) # type: ignore[no-any-return, operator] diff --git a/src/test_workflow/integ_test/local_test_cluster.py b/src/test_workflow/integ_test/local_test_cluster.py index 35b7306798..11b5d7ba6c 100644 --- a/src/test_workflow/integ_test/local_test_cluster.py +++ b/src/test_workflow/integ_test/local_test_cluster.py @@ -40,6 +40,7 @@ def __init__( self.service_opensearch = ServiceOpenSearch( self.manifest.build.version, + self.manifest.build.distribution, self.additional_cluster_config, self.security_enabled, self.dependency_installer, diff --git a/src/test_workflow/integ_test/local_test_cluster_opensearch_dashboards.py b/src/test_workflow/integ_test/local_test_cluster_opensearch_dashboards.py index 4ec5097403..23e147e912 100644 --- a/src/test_workflow/integ_test/local_test_cluster_opensearch_dashboards.py +++ b/src/test_workflow/integ_test/local_test_cluster_opensearch_dashboards.py @@ -45,15 +45,15 @@ def __init__( self.service_opensearch = ServiceOpenSearch( self.manifest_opensearch.build.version, + self.manifest_opensearch.build.distribution, {}, self.security_enabled, self.dependency_installer_opensearch, self.work_dir) - build = self.manifest_opensearch_dashboards.build - self.service_opensearch_dashboards = ServiceOpenSearchDashboards( - build.version, + self.manifest_opensearch_dashboards.build.version, + self.manifest_opensearch_dashboards.build.distribution, self.additional_cluster_config, self.security_enabled, self.dependency_installer_opensearch_dashboards, diff --git a/src/test_workflow/integ_test/service.py b/src/test_workflow/integ_test/service.py index 41a2708034..887b33ed2e 100644 --- a/src/test_workflow/integ_test/service.py +++ b/src/test_workflow/integ_test/service.py @@ -7,6 +7,7 @@ import abc import logging import time +import os import requests @@ -19,9 +20,10 @@ class Service(abc.ABC): Abstract base class for all types of test clusters. """ - def __init__(self, work_dir, version, security_enabled, additional_config, dependency_installer): + def __init__(self, work_dir, version, distribution, security_enabled, additional_config, dependency_installer): self.work_dir = work_dir self.version = version + self.distribution = distribution self.security_enabled = security_enabled self.additional_config = additional_config self.dependency_installer = dependency_installer @@ -43,6 +45,8 @@ def terminate(self): self.return_code = self.process_handler.terminate() + self.cleanup() + return ServiceTerminationResult(self.return_code, self.process_handler.stdout_data, self.process_handler.stderr_data, self.log_files) def endpoint(self): @@ -80,6 +84,12 @@ def service_alive(self): else: return False + def download(self): + logging.info("Downloading bundle artifact") + bundle_name = self.dependency_installer.download_dist(self.work_dir) + logging.info(f"Downloaded bundle to {os.path.realpath(bundle_name)}") + return bundle_name + def wait_for_service(self): logging.info("Waiting for service to become available") diff --git a/src/test_workflow/integ_test/service_opensearch.py b/src/test_workflow/integ_test/service_opensearch.py index 4f3ec7386c..b7d936e753 100644 --- a/src/test_workflow/integ_test/service_opensearch.py +++ b/src/test_workflow/integ_test/service_opensearch.py @@ -12,27 +12,29 @@ import yaml from test_workflow.integ_test.service import Service +from test_workflow.integ_test.distributions import Distributions class ServiceOpenSearch(Service): def __init__( self, version, + distribution, additional_config, security_enabled, dependency_installer, work_dir ): - super().__init__(work_dir, version, security_enabled, additional_config, dependency_installer) + super().__init__(work_dir, version, distribution, security_enabled, additional_config, dependency_installer) + self.dist = Distributions.get_distribution("opensearch", distribution, version, work_dir) self.dependency_installer = dependency_installer - - self.install_dir = os.path.join(self.work_dir, f"opensearch-{self.version}") + self.install_dir = self.dist.get_install_dir def start(self): - self.__download() + self.dist.install_distribution(self.download()) - self.opensearch_yml_dir = os.path.join(self.install_dir, "config", "opensearch.yml") + self.opensearch_yml_dir = os.path.join(self.dist.get_config_dir, "opensearch.yml") self.security_plugin_dir = os.path.join(self.install_dir, "plugins", "opensearch-security") if not self.security_enabled and os.path.isdir(self.security_plugin_dir): @@ -41,20 +43,11 @@ def start(self): if self.additional_config: self.__add_plugin_specific_config(self.additional_config) - self.process_handler.start("./opensearch-tar-install.sh", self.install_dir) + self.process_handler.start(self.dist.get_start_cmd, self.install_dir) logging.info(f"Started OpenSearch with parent PID {self.process_handler.pid}") - def __download(self): - logging.info(f"Creating local test cluster in {self.work_dir}") - logging.info("Downloading bundle") - bundle_name = self.dependency_installer.download_dist(self.work_dir) - logging.info(f"Downloaded bundle to {os.path.realpath(bundle_name)}") - - logging.info(f"Unpacking {bundle_name}") - with tarfile.open(bundle_name, 'r') as bundle_tar: - bundle_tar.extractall(self.work_dir) - - logging.info(f"Unpacked {bundle_name}") + def cleanup(self): + self.dist.cleanup() def url(self, path=""): return f'{"https" if self.security_enabled else "http"}://{self.endpoint()}:{self.port()}{path}' diff --git a/src/test_workflow/integ_test/service_opensearch_dashboards.py b/src/test_workflow/integ_test/service_opensearch_dashboards.py index 7e9744ff55..c4dfcb5d4d 100644 --- a/src/test_workflow/integ_test/service_opensearch_dashboards.py +++ b/src/test_workflow/integ_test/service_opensearch_dashboards.py @@ -13,25 +13,27 @@ import yaml from test_workflow.integ_test.service import Service +from test_workflow.integ_test.distributions import Distributions class ServiceOpenSearchDashboards(Service): def __init__( self, version, + distribution, additional_config, security_enabled, dependency_installer, work_dir ): - super().__init__(work_dir, version, security_enabled, additional_config, dependency_installer) - self.install_dir = os.path.join(self.work_dir, f"opensearch-dashboards-{self.version}") + super().__init__(work_dir, version, distribution, security_enabled, additional_config, dependency_installer) + self.dist = Distributions.get_distribution("opensearch-dashboards", distribution, version, work_dir) + self.install_dir = self.dist.get_install_dir def start(self): - logging.info(f"Starting OpenSearch Dashboards service from {self.work_dir}") - self.__download() + self.dist.install_distribution(self.download()) - self.opensearch_dashboards_yml_dir = os.path.join(self.install_dir, "config", "opensearch_dashboards.yml") + self.opensearch_dashboards_yml_dir = os.path.join(self.dist.get_config_dir, "opensearch_dashboards.yml") self.executable_dir = os.path.join(self.install_dir, "bin") if not self.security_enabled: @@ -42,8 +44,11 @@ def start(self): if self.additional_config: self.__add_plugin_specific_config(self.additional_config) - self.process_handler.start("./opensearch-dashboards", self.executable_dir) - logging.info(f"Started OpenSearch Dashboards with parent PID {self.process_handler.pid}") + self.process_handler.start(self.dist.get_start_cmd, self.executable_dir) + logging.info(f"Started OpenSearch with parent PID {self.process_handler.pid}") + + def cleanup(self): + self.dist.cleanup() def __set_logging_dest(self): self.log_dir = os.path.join(self.install_dir, "logs") @@ -53,22 +58,11 @@ def __set_logging_dest(self): def __remove_security(self): self.security_plugin_dir = os.path.join(self.install_dir, "plugins", "securityDashboards") if os.path.isdir(self.security_plugin_dir): - subprocess.check_call("./opensearch-dashboards-plugin remove securityDashboards", cwd=self.executable_dir, shell=True) + subprocess.check_call("./opensearch-dashboards-plugin remove --allow-root securityDashboards", cwd=self.executable_dir, shell=True) with open(self.opensearch_dashboards_yml_dir, "w") as yamlfile: yamlfile.close() - def __download(self): - logging.info("Downloading OpenSearch Dashboards bundle") - bundle_name = self.dependency_installer.download_dist(self.work_dir) - logging.info(f"Downloaded bundle to {os.path.realpath(bundle_name)}") - - logging.info(f"Unpacking {bundle_name}") - with tarfile.open(bundle_name, 'r') as bundle_tar: - bundle_tar.extractall(self.work_dir) - - logging.info(f"Unpacked {bundle_name}") - def url(self, path=""): return f'http://{self.endpoint()}:{self.port()}{path}' From a2665781512840035a0698997754c76b0939ea5e Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 19 Apr 2022 23:30:48 +0000 Subject: [PATCH 2/8] Add syntax checks Signed-off-by: Peter Zhu --- src/test_workflow/integ_test/distribution.py | 10 +--------- src/test_workflow/integ_test/distribution_rpm.py | 7 ++----- src/test_workflow/integ_test/distribution_tar.py | 9 +++------ src/test_workflow/integ_test/distributions.py | 6 +++--- src/test_workflow/integ_test/service.py | 2 +- src/test_workflow/integ_test/service_opensearch.py | 3 +-- .../integ_test/service_opensearch_dashboards.py | 3 +-- 7 files changed, 12 insertions(+), 28 deletions(-) diff --git a/src/test_workflow/integ_test/distribution.py b/src/test_workflow/integ_test/distribution.py index 7829e033aa..e8cc447bfd 100644 --- a/src/test_workflow/integ_test/distribution.py +++ b/src/test_workflow/integ_test/distribution.py @@ -4,15 +4,8 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -import errno -import logging -import os -import shutil -import subprocess from abc import ABC, abstractmethod -from typing import Any, List -from system.process import Process class Distribution(ABC): def __enter__(self) -> 'Distribution': @@ -41,7 +34,7 @@ def get_config_dir(self, bundle_name: str) -> str: pass @abstractmethod - def install_distribution(self) ->None: + def install_distribution(self) -> None: """ The detailed method to install the distribution before start the service """ @@ -60,4 +53,3 @@ def cleanup(self) -> None: Allow distribution that is not 'tar' to do proper cleanup """ pass - diff --git a/src/test_workflow/integ_test/distribution_rpm.py b/src/test_workflow/integ_test/distribution_rpm.py index a6987e607d..9806f69ae2 100644 --- a/src/test_workflow/integ_test/distribution_rpm.py +++ b/src/test_workflow/integ_test/distribution_rpm.py @@ -4,16 +4,13 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -import errno import logging import os -import tarfile -import shutil import subprocess -from system.process import Process from test_workflow.integ_test.distribution import Distribution + class DistributionRpm(Distribution): def __init__(self, filename: str, distribution: str, version: str, work_dir: str) -> None: super().__init__(filename, distribution, version, work_dir) @@ -26,7 +23,7 @@ def get_install_dir(self) -> str: def get_config_dir(self) -> str: return os.path.join(os.sep, "etc", self.filename) - def install_distribution(self, bundle_name: str) ->None: + def install_distribution(self, bundle_name: str) -> None: logging.info(f"Installing {bundle_name} in {self.get_install_dir}") logging.info("rpm installation requires sudo, script will exit if current user does not have sudo access") rpm_install_cmd = " ".join( diff --git a/src/test_workflow/integ_test/distribution_tar.py b/src/test_workflow/integ_test/distribution_tar.py index 39a26f5f45..0771ece791 100644 --- a/src/test_workflow/integ_test/distribution_tar.py +++ b/src/test_workflow/integ_test/distribution_tar.py @@ -4,15 +4,13 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. -import errno import logging import os import tarfile -import shutil -import subprocess from test_workflow.integ_test.distribution import Distribution + class DistributionTar(Distribution): def __init__(self, filename: str, distribution: str, version: str, work_dir: str) -> None: super().__init__(filename, distribution, version, work_dir) @@ -25,10 +23,10 @@ def get_install_dir(self) -> str: def get_config_dir(self) -> str: return os.path.join(self.get_install_dir, "config") - def install_distribution(self, bundle_name: str) ->None: + def install_distribution(self, bundle_name: str) -> None: logging.info(f"Installing {bundle_name} in {self.get_install_dir}") with tarfile.open(bundle_name, 'r') as bundle_tar: - bundle_tar.extractall(self.work_dir) + bundle_tar.extractall(self.work_dir) @property def get_start_cmd(self) -> str: @@ -37,4 +35,3 @@ def get_start_cmd(self) -> str: "opensearch-dashboards": "./opensearch-dashboards", } return start_cmd_map[self.filename] - diff --git a/src/test_workflow/integ_test/distributions.py b/src/test_workflow/integ_test/distributions.py index a47b90008f..8c9a8bbf6d 100644 --- a/src/test_workflow/integ_test/distributions.py +++ b/src/test_workflow/integ_test/distributions.py @@ -7,8 +7,8 @@ import logging from test_workflow.integ_test.distribution import Distribution -from test_workflow.integ_test.distribution_tar import DistributionTar from test_workflow.integ_test.distribution_rpm import DistributionRpm +from test_workflow.integ_test.distribution_tar import DistributionTar class Distributions: @@ -22,10 +22,10 @@ def from_name(cls, name: str) -> Distribution: klass = cls.DISTRIBUTION_MAP.get(name, None) if not klass: raise ValueError(f"Unsupported distribution: {name}") - return klass # type: ignore[return-value] + return klass # type: ignore[return-value] @classmethod def get_distribution(cls, filename: str, distribution: str, version: str, work_dir: str) -> Distribution: klass = cls.from_name(distribution) logging.info(f"{filename} distribution: {distribution}") - return klass(filename, distribution, version, work_dir) # type: ignore[no-any-return, operator] + return klass(filename, distribution, version, work_dir) # type: ignore[no-any-return, operator] diff --git a/src/test_workflow/integ_test/service.py b/src/test_workflow/integ_test/service.py index 887b33ed2e..44c60f83f2 100644 --- a/src/test_workflow/integ_test/service.py +++ b/src/test_workflow/integ_test/service.py @@ -6,8 +6,8 @@ import abc import logging -import time import os +import time import requests diff --git a/src/test_workflow/integ_test/service_opensearch.py b/src/test_workflow/integ_test/service_opensearch.py index b7d936e753..6a6a5fb43f 100644 --- a/src/test_workflow/integ_test/service_opensearch.py +++ b/src/test_workflow/integ_test/service_opensearch.py @@ -6,13 +6,12 @@ import logging import os -import tarfile import requests import yaml -from test_workflow.integ_test.service import Service from test_workflow.integ_test.distributions import Distributions +from test_workflow.integ_test.service import Service class ServiceOpenSearch(Service): diff --git a/src/test_workflow/integ_test/service_opensearch_dashboards.py b/src/test_workflow/integ_test/service_opensearch_dashboards.py index c4dfcb5d4d..25492425b6 100644 --- a/src/test_workflow/integ_test/service_opensearch_dashboards.py +++ b/src/test_workflow/integ_test/service_opensearch_dashboards.py @@ -7,13 +7,12 @@ import logging import os import subprocess -import tarfile import requests import yaml -from test_workflow.integ_test.service import Service from test_workflow.integ_test.distributions import Distributions +from test_workflow.integ_test.service import Service class ServiceOpenSearchDashboards(Service): From 371d1667951a19d55765217c823b4d147c3922ed Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 20 Apr 2022 01:14:49 +0000 Subject: [PATCH 3/8] Update tests Signed-off-by: Peter Zhu --- src/test_workflow/integ_test/distribution.py | 3 +- .../integ_test/distribution_rpm.py | 4 +- .../integ_test/distribution_tar.py | 6 +-- src/test_workflow/integ_test/distributions.py | 6 +-- .../.test_bundle_rpm.py.swp | Bin 0 -> 16384 bytes .../opensearch-min-1.3.0-linux-x64.tar.gz | Bin 0 -> 223 bytes .../integ_test/test_distribution_rpm.py | 51 ++++++++++++++++++ .../integ_test/test_distribution_tar.py | 45 ++++++++++++++++ .../integ_test/test_distributions.py | 27 ++++++++++ .../integ_test/test_local_test_cluster.py | 2 + ...ocal_test_cluster_opensearch_dashboards.py | 4 ++ .../integ_test/test_service_opensearch.py | 20 ++++++- .../test_service_opensearch_dashboards.py | 14 ++++- 13 files changed, 169 insertions(+), 13 deletions(-) create mode 100644 tests/tests_assemble_workflow/.test_bundle_rpm.py.swp create mode 100644 tests/tests_test_workflow/test_integ_workflow/integ_test/data/artifacts/dist/opensearch-min-1.3.0-linux-x64.tar.gz create mode 100644 tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py create mode 100644 tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py create mode 100644 tests/tests_test_workflow/test_integ_workflow/integ_test/test_distributions.py diff --git a/src/test_workflow/integ_test/distribution.py b/src/test_workflow/integ_test/distribution.py index e8cc447bfd..195e83af65 100644 --- a/src/test_workflow/integ_test/distribution.py +++ b/src/test_workflow/integ_test/distribution.py @@ -11,9 +11,8 @@ class Distribution(ABC): def __enter__(self) -> 'Distribution': return self - def __init__(self, filename: str, distribution: str, version: str, work_dir: str) -> None: + def __init__(self, filename: str, version: str, work_dir: str) -> None: self.filename = filename - self.distribution = distribution self.version = version self.work_dir = work_dir diff --git a/src/test_workflow/integ_test/distribution_rpm.py b/src/test_workflow/integ_test/distribution_rpm.py index 9806f69ae2..db5361c252 100644 --- a/src/test_workflow/integ_test/distribution_rpm.py +++ b/src/test_workflow/integ_test/distribution_rpm.py @@ -12,8 +12,8 @@ class DistributionRpm(Distribution): - def __init__(self, filename: str, distribution: str, version: str, work_dir: str) -> None: - super().__init__(filename, distribution, version, work_dir) + def __init__(self, filename: str, version: str, work_dir: str) -> None: + super().__init__(filename, version, work_dir) @property def get_install_dir(self) -> str: diff --git a/src/test_workflow/integ_test/distribution_tar.py b/src/test_workflow/integ_test/distribution_tar.py index 0771ece791..5c52b50866 100644 --- a/src/test_workflow/integ_test/distribution_tar.py +++ b/src/test_workflow/integ_test/distribution_tar.py @@ -12,8 +12,8 @@ class DistributionTar(Distribution): - def __init__(self, filename: str, distribution: str, version: str, work_dir: str) -> None: - super().__init__(filename, distribution, version, work_dir) + def __init__(self, filename: str, version: str, work_dir: str) -> None: + super().__init__(filename, version, work_dir) @property def get_install_dir(self) -> str: @@ -25,7 +25,7 @@ def get_config_dir(self) -> str: def install_distribution(self, bundle_name: str) -> None: logging.info(f"Installing {bundle_name} in {self.get_install_dir}") - with tarfile.open(bundle_name, 'r') as bundle_tar: + with tarfile.open(bundle_name, 'r:gz') as bundle_tar: bundle_tar.extractall(self.work_dir) @property diff --git a/src/test_workflow/integ_test/distributions.py b/src/test_workflow/integ_test/distributions.py index 8c9a8bbf6d..739e535496 100644 --- a/src/test_workflow/integ_test/distributions.py +++ b/src/test_workflow/integ_test/distributions.py @@ -12,14 +12,14 @@ class Distributions: - DISTRIBUTION_MAP = { + DISTRIBUTIONS_MAP = { "tar": DistributionTar, "rpm": DistributionRpm, } @classmethod def from_name(cls, name: str) -> Distribution: - klass = cls.DISTRIBUTION_MAP.get(name, None) + klass = cls.DISTRIBUTIONS_MAP.get(name, None) if not klass: raise ValueError(f"Unsupported distribution: {name}") return klass # type: ignore[return-value] @@ -28,4 +28,4 @@ def from_name(cls, name: str) -> Distribution: def get_distribution(cls, filename: str, distribution: str, version: str, work_dir: str) -> Distribution: klass = cls.from_name(distribution) logging.info(f"{filename} distribution: {distribution}") - return klass(filename, distribution, version, work_dir) # type: ignore[no-any-return, operator] + return klass(filename, version, work_dir) # type: ignore[no-any-return, operator] diff --git a/tests/tests_assemble_workflow/.test_bundle_rpm.py.swp b/tests/tests_assemble_workflow/.test_bundle_rpm.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..b8e8655eb69b5d1ce616724f66240cd99fa72fa8 GIT binary patch literal 16384 zcmeHNNpBoQ6fOc`3jsnd95@t{ps}Utwmpfh$SA8ZN`SUEN-S zfEzi$EfR3Z9d6vh4`4gwNDhc!0NjBa@KrA}(>>!whKNX1C!c4!x|Ubpt5@}2&ufie zy>OA9Fq;I|^Mu^LbFFp%-choAl#rX9WJRzIp&o5_Ieg7<8BH=4|toSJGLpE%CV zL>lW{#zxa5?-v!uf{^=?v&ilktwgwv5%QQvc+5O@Gw{b^w2;|fi%H47mdma6AX;s^ z!FpP5wG!V!VHA31xVfEl`k7`xGmtTGkQ{4HjH~ZQYA@5nFI>^YXa+O`ngPv#W zGy|Fe&46Y=Gw={Hz#Kwegk0BDz>nYmi}?Si=Lq=`xC69+Gr(zJ5_kpp>sdnn1nvQ! z0iOb&06yRW^S~T%7`S_o5DqxN0pQj%gnS6R1l&4+@4yN$4@?4wfRCRhSED#X}LniDs{oSkuX=Z z1&@}-m$Uz)q-ZI*+cs79Lln=ddL!)`nxy#>)M&M+VK}@kd`|0D9E6UD=(|~yC>CvI z$I=Q}+&v6iX?|@%Ej>wxO4P01?I5MEmsrx%j+wGP zl0FZay~?ihl-vd#!gxbTt*AFw0^w(kSV<=hTJKSc%oymx$aOKAcj+--w2=iaDiMs?oP^dj_u5jsgMlycU1s-dD>brcI zNmO|TmX%)$LFON=8F?&RTuB(X-p7iJcvRr8Ymmb3-C4Zy1fkWsskB?YZTOd)^!ai1W zSj+~KZ$WX%2{AW4w^lWoVQx_>l1jr}fRg%L8m#PrRar8~(s1>@H8wE+2RrL@hu>kk zd!J-}e)KG8fE4sysB3FqvD8Ybw7RpDe9LN=MgLnahpuZn=D?ZDmoUXOFfO|8Q?ijx z4}Y^LK}81jY^usJl{~%@riR;RTf}w|&(K9Ut>@D8ZMAL^U&P9dHC6EplYDd<@swKJ zv@isjowrtTPlXo-QCzl%tjR#OIGPrT;=N}1n$3slN}!^ZqEhB5r%o)%hb!t8_kl!g zO6|ZyErAU6G)|&2NFtk4SJ*1Y-9(MJBP3NuNL$@yQ08zN2jmD^U!H&Mn(NUQF@Q)T+n3}^;41DXNNfM!55pc&8%Xa+O`ngPv# aX5e9AK)tU~SNhJT|2O06OVH{MT7Lr@C1q;> literal 0 HcmV?d00001 diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/data/artifacts/dist/opensearch-min-1.3.0-linux-x64.tar.gz b/tests/tests_test_workflow/test_integ_workflow/integ_test/data/artifacts/dist/opensearch-min-1.3.0-linux-x64.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..afc2a9438a123ab9610531ca431c9f04eb440020 GIT binary patch literal 223 zcmV<503iP#iwFR&0uy2Y1MSg4Zo)7Sg<-Ebg)cywA#rk%UceMX+@=m9Mxv}xFV?H! zAP`bZ)lEtl{lArG#?p-Ry0r_vvzw=8Dp`@`4{CgQQX8WVL-Fyx7-}iYQ~yj!DKe9) zu4JatFgK#K`Gs4)*!IzGv{w6O`|Rz`fA`P6+I<)9anW6MpYy*qvbc+*>=**BQr?H5 zxRrl;U-CCnr2MP8sFlt~+5Urnp8sRxd%d None: + + self.work_dir = os.path.join(os.path.dirname(__file__), "data") + self.distribution_rpm = DistributionRpm("opensearch", "1.3.0", self.work_dir) + self.distribution_rpm_dashboards = DistributionRpm("opensearch-dashboards", "1.3.0", self.work_dir) + + def test_distribution_rpm_vars(self) -> None: + self.assertEqual(self.distribution_rpm.filename, 'opensearch') + self.assertEqual(self.distribution_rpm.version, '1.3.0') + self.assertEqual(self.distribution_rpm.work_dir, self.work_dir) + + def test_get_install_dir(self) -> None: + self.assertEqual(self.distribution_rpm.get_install_dir, os.path.join(os.sep, "usr", "share", "opensearch")) + + def test_get_config_dir(self) -> None: + self.assertEqual(self.distribution_rpm.get_config_dir, os.path.join(os.sep, "etc", "opensearch")) + + @patch("subprocess.check_call") + def test_install_distribution(self, check_call_mock: Mock) -> None: + self.distribution_rpm.install_distribution("opensearch.rpm") + args_list = check_call_mock.call_args_list + + self.assertEqual(check_call_mock.call_count, 1) + self.assertEqual("yum remove -y opensearch && yum install -y opensearch.rpm", args_list[0][0][0]) + + def test_get_srpmt_cmd(self) -> None: + self.assertEqual(self.distribution_rpm.get_start_cmd, "systemctl start opensearch") + self.assertEqual(self.distribution_rpm_dashboards.get_start_cmd, "systemctl start opensearch-dashboards") + + @patch("subprocess.check_call") + def test_cleanup(self, check_call_mock: Mock) -> None: + self.distribution_rpm.cleanup() + args_list = check_call_mock.call_args_list + + self.assertEqual(check_call_mock.call_count, 1) + self.assertEqual("yum remove -y 'opensearch*'", args_list[0][0][0]) diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py new file mode 100644 index 0000000000..c163e3953d --- /dev/null +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +import unittest + +from test_workflow.integ_test.distribution_tar import DistributionTar +from unittest.mock import MagicMock, patch + + +class TestDistributionTar(unittest.TestCase): + + def setUp(self) -> None: + + self.work_dir = os.path.join(os.path.dirname(__file__), "data") + self.distribution_tar = DistributionTar("opensearch", "1.3.0", self.work_dir) + self.distribution_tar_dashboards = DistributionTar("opensearch-dashboards", "1.3.0", self.work_dir) + + def test_distribution_tar_vars(self) -> None: + self.assertEqual(self.distribution_tar.filename, 'opensearch') + self.assertEqual(self.distribution_tar.version, '1.3.0') + self.assertEqual(self.distribution_tar.work_dir, self.work_dir) + + def test_get_install_dir(self) -> None: + self.assertEqual(self.distribution_tar.get_install_dir, os.path.join(self.work_dir, "opensearch-1.3.0")) + + def test_get_config_dir(self) -> None: + self.assertEqual(self.distribution_tar.get_config_dir, os.path.join(self.work_dir, "opensearch-1.3.0", "config")) + + def test_install_distribution(self) -> None: + with patch("tarfile.open") as mock_tarfile_open: + mock_tarfile_extractall = MagicMock() + mock_tarfile_open.return_value.__enter__.return_value.extractall = mock_tarfile_extractall + + self.distribution_tar.install_distribution(os.path.join(self.work_dir, "artifacts", "dist", "opensearch-min-1.3.0-linux-x64.tar.gz")) + + mock_tarfile_open.assert_called_with(os.path.join(self.work_dir, "artifacts", "dist", "opensearch-min-1.3.0-linux-x64.tar.gz"), "r:gz") + mock_tarfile_extractall.assert_called_with(self.work_dir) + + def test_get_start_cmd(self) -> None: + self.assertEqual(self.distribution_tar.get_start_cmd, "./opensearch-tar-install.sh") + self.assertEqual(self.distribution_tar_dashboards.get_start_cmd, "./opensearch-dashboards") diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distributions.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distributions.py new file mode 100644 index 0000000000..3bebb7fce7 --- /dev/null +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distributions.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +import unittest + +from test_workflow.integ_test.distributions import Distributions + + +class TestDistributions(unittest.TestCase): + + def setUp(self) -> None: + + self.distributions = Distributions + + def test_distribution_map(self) -> None: + self.assertEqual(self.distributions.DISTRIBUTIONS_MAP['tar'].__name__, 'DistributionTar') + self.assertEqual(self.distributions.DISTRIBUTIONS_MAP['rpm'].__name__, 'DistributionRpm') + + def test_create_dist(self) -> None: + return_cls_tar = self.distributions.get_distribution("opensearch", "tar", "1.3.0", os.path.join(os.path.dirname(__file__), "data")) + self.assertEqual(return_cls_tar.__class__.__name__, 'DistributionTar') + return_cls_rpm = self.distributions.get_distribution("opensearch", "rpm", "1.3.0", os.path.join(os.path.dirname(__file__), "data")) + self.assertEqual(return_cls_rpm.__class__.__name__, 'DistributionRpm') diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster.py index 339107e92a..392c453e39 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster.py @@ -18,6 +18,7 @@ class LocalTestClusterTests(unittest.TestCase): def setUp(self): mock_manifest = MagicMock() mock_manifest.build.version = "1.1.0" + mock_manifest.build.distribution = "tar" self.manifest = mock_manifest self.work_dir = "test_work_dir" @@ -54,6 +55,7 @@ def test_start(self, mock_service): mock_service.assert_called_once_with( "1.1.0", + "tar", self.additional_cluster_config, self.security_enabled, self.dependency_installer, diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster_opensearch_dashboards.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster_opensearch_dashboards.py index 335ca34f70..39f2447e39 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster_opensearch_dashboards.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_local_test_cluster_opensearch_dashboards.py @@ -18,10 +18,12 @@ class LocalTestClusterOpenSearchDashboardsTests(unittest.TestCase): def setUp(self): mock_bundle_manifest_opensearch = MagicMock() mock_bundle_manifest_opensearch.build.version = "1.1.0" + mock_bundle_manifest_opensearch.build.distribution = "tar" self.mock_bundle_manifest_opensearch = mock_bundle_manifest_opensearch mock_bundle_manifest_opensearch_dashboards = MagicMock() mock_bundle_manifest_opensearch_dashboards.build.version = "1.1.0" + mock_bundle_manifest_opensearch_dashboards.build.distribution = "tar" self.mock_bundle_manifest_opensearch_dashboards = mock_bundle_manifest_opensearch_dashboards dependency_installer_opensearch = MagicMock() @@ -70,6 +72,7 @@ def test_start(self, mock_service_opensearch_dashboards, mock_service_opensearch mock_service_opensearch.assert_called_once_with( "1.1.0", + "tar", {}, self.security_enabled, self.dependency_installer_opensearch, @@ -81,6 +84,7 @@ def test_start(self, mock_service_opensearch_dashboards, mock_service_opensearch mock_service_opensearch_dashboards.assert_called_once_with( "1.1.0", + "tar", self.additional_cluster_config, self.security_enabled, self.dependency_installer_opensearch_dashboards, diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py index f6dfa07973..3900558c35 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch.py @@ -17,6 +17,7 @@ class ServiceOpenSearchTests(unittest.TestCase): def setUp(self): self.version = "1.1.0" + self.distribution = "tar" self.work_dir = "test_work_dir" self.additional_config = {"script.context.field.max_compilations_rate": "1000/1m"} self.dependency_installer = "" @@ -33,6 +34,7 @@ def test_start(self, mock_tarfile_open, mock_dump, mock_file, mock_pid, mock_pro service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, dependency_installer, @@ -59,7 +61,7 @@ def test_start(self, mock_tarfile_open, mock_dump, mock_file, mock_pid, mock_pro mock_file.return_value.write.assert_called_once_with(mock_dump_result) dependency_installer.download_dist.assert_called_once_with(self.work_dir) - mock_tarfile_open.assert_called_once_with(bundle_full_name, "r") + mock_tarfile_open.assert_called_once_with(bundle_full_name, "r:gz") mock_bundle_tar.extractall.assert_called_once_with(self.work_dir) self.assertEqual(mock_pid.call_count, 1) @@ -76,6 +78,7 @@ def test_start_security_disabled(self, mock_tarfile_open, mock_dump, mock_file, service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, False, dependency_installer, @@ -125,6 +128,7 @@ def test_start_security_disabled_and_not_installed(self, mock_tarfile_open, mock service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, False, dependency_installer, @@ -174,6 +178,7 @@ def test_terminate( ): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -194,6 +199,7 @@ def test_terminate( def test_terminate_process_not_started(self, mock_process_started, mock_process_terminate): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -208,6 +214,7 @@ def test_terminate_process_not_started(self, mock_process_started, mock_process_ def test_endpoint_port(self): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -220,6 +227,7 @@ def test_endpoint_port(self): def test_url_security_enabled(self): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -231,6 +239,7 @@ def test_url_security_enabled(self): def test_url_security_disabled(self): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, False, self.dependency_installer, @@ -244,6 +253,7 @@ def test_url_security_disabled(self): def test_get_service_response(self, mock_url, mock_requests_get): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -267,6 +277,7 @@ def test_get_service_response(self, mock_url, mock_requests_get): def test_wait_for_service_succeed_on_first_attemp(self, mock_process_stderr_data, mock_process_stdout_data, mock_service_alive, mock_time_sleep): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -293,6 +304,7 @@ def test_wait_for_service_always_fail_without_exception( ): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -323,6 +335,7 @@ def test_wait_for_service_always_fail_with_exception( service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -355,6 +368,7 @@ def test_wait_for_service_suceed_on_third_attempt( ): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -372,6 +386,7 @@ def test_wait_for_service_suceed_on_third_attempt( def test_service_alive_green_available(self, mock_get_service_response): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -391,6 +406,7 @@ def test_service_alive_green_available(self, mock_get_service_response): def test_service_alive_yellow_available(self, mock_get_service_response): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -410,6 +426,7 @@ def test_service_alive_yellow_available(self, mock_get_service_response): def test_service_alive_red_unavailable(self, mock_get_service_response): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -429,6 +446,7 @@ def test_service_alive_red_unavailable(self, mock_get_service_response): def test_service_alive_unavailable(self, mock_get_service_response): service = ServiceOpenSearch( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch_dashboards.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch_dashboards.py index 6a48cb6632..e8ce017ff1 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch_dashboards.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_service_opensearch_dashboards.py @@ -15,6 +15,7 @@ class ServiceOpenSearchDashboardsTests(unittest.TestCase): def setUp(self): self.version = "1.1.0" + self.distribution = "tar" self.work_dir = "test_work_dir" self.additional_config = {"script.context.field.max_compilations_rate": "1000/1m"} self.dependency_installer = "" @@ -30,6 +31,7 @@ def test_start(self, mock_tarfile_open, mock_dump, mock_file, mock_pid, mock_pro service = ServiceOpenSearchDashboards( self.version, + self.distribution, self.additional_config, True, mock_dependency_installer, @@ -49,7 +51,7 @@ def test_start(self, mock_tarfile_open, mock_dump, mock_file, mock_pid, mock_pro service.start() mock_dependency_installer.download_dist.called_once_with(self.work_dir) - mock_tarfile_open.assert_called_once_with(bundle_full_name, "r") + mock_tarfile_open.assert_called_once_with(bundle_full_name, "r:gz") mock_bundle_tar.extractall.assert_called_once_with(self.work_dir) mock_file.assert_called_once_with(os.path.join(self.work_dir, "opensearch-dashboards-1.1.0", "config", "opensearch_dashboards.yml"), "a") @@ -77,6 +79,7 @@ def test_start_without_security(self, mock_tarfile_open, mock_dump, mock_file, m service = ServiceOpenSearchDashboards( self.version, + self.distribution, {}, False, mock_dependency_installer, @@ -109,7 +112,7 @@ def test_start_without_security(self, mock_tarfile_open, mock_dump, mock_file, m ) mock_check_call.assert_called_once_with( - "./opensearch-dashboards-plugin remove securityDashboards", + "./opensearch-dashboards-plugin remove --allow-root securityDashboards", cwd=os.path.join("test_work_dir", "opensearch-dashboards-1.1.0", "bin"), shell=True ) @@ -133,6 +136,7 @@ def test_start_without_security_and_not_installed(self, mock_tarfile_open, mock_ service = ServiceOpenSearchDashboards( self.version, + self.distribution, {}, False, mock_dependency_installer, @@ -175,6 +179,7 @@ def test_start_without_security_and_not_installed(self, mock_tarfile_open, mock_ def test_endpoint_port_url(self): service = ServiceOpenSearchDashboards( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -190,6 +195,7 @@ def test_endpoint_port_url(self): def test_get_service_response_with_security(self, mock_url, mock_requests_get): service = ServiceOpenSearchDashboards( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -209,6 +215,7 @@ def test_get_service_response_with_security(self, mock_url, mock_requests_get): def test_get_service_response_without_security(self, mock_url, mock_requests_get): service = ServiceOpenSearchDashboards( self.version, + self.distribution, self.additional_config, False, self.dependency_installer, @@ -227,6 +234,7 @@ def test_get_service_response_without_security(self, mock_url, mock_requests_get def test_service_alive_green_available(self, mock_get_service_response): service = ServiceOpenSearchDashboards( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -246,6 +254,7 @@ def test_service_alive_green_available(self, mock_get_service_response): def test_service_alive_yellow_available(self, mock_get_service_response): service = ServiceOpenSearchDashboards( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, @@ -265,6 +274,7 @@ def test_service_alive_yellow_available(self, mock_get_service_response): def test_service_alive_red_unavailable(self, mock_get_service_response): service = ServiceOpenSearchDashboards( self.version, + self.distribution, self.additional_config, True, self.dependency_installer, From 5bdbe1fd2b83a67fc3b9082475d68cf7dc5998f5 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 20 Apr 2022 01:25:11 +0000 Subject: [PATCH 4/8] Fix isort Signed-off-by: Peter Zhu --- .../test_integ_workflow/integ_test/test_distribution_rpm.py | 2 +- .../test_integ_workflow/integ_test/test_distribution_tar.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py index 9d17d7d098..6c0a5025d8 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py @@ -6,9 +6,9 @@ import os import unittest +from unittest.mock import Mock, patch from test_workflow.integ_test.distribution_rpm import DistributionRpm -from unittest.mock import Mock, patch class TestDistributionRpm(unittest.TestCase): diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py index c163e3953d..f4601a3d4d 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py @@ -6,9 +6,9 @@ import os import unittest +from unittest.mock import MagicMock, patch from test_workflow.integ_test.distribution_tar import DistributionTar -from unittest.mock import MagicMock, patch class TestDistributionTar(unittest.TestCase): From 64ec24d61b4ff39153c2bc779423a336fd8ed770 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 20 Apr 2022 02:08:51 +0000 Subject: [PATCH 5/8] More typo fixes Signed-off-by: Peter Zhu --- .../integ_test/service_opensearch_dashboards.py | 2 +- .../.test_bundle_rpm.py.swp | Bin 16384 -> 0 bytes 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 tests/tests_assemble_workflow/.test_bundle_rpm.py.swp diff --git a/src/test_workflow/integ_test/service_opensearch_dashboards.py b/src/test_workflow/integ_test/service_opensearch_dashboards.py index 25492425b6..2da51da30f 100644 --- a/src/test_workflow/integ_test/service_opensearch_dashboards.py +++ b/src/test_workflow/integ_test/service_opensearch_dashboards.py @@ -44,7 +44,7 @@ def start(self): self.__add_plugin_specific_config(self.additional_config) self.process_handler.start(self.dist.get_start_cmd, self.executable_dir) - logging.info(f"Started OpenSearch with parent PID {self.process_handler.pid}") + logging.info(f"Started OpenSearch Dashboards with parent PID {self.process_handler.pid}") def cleanup(self): self.dist.cleanup() diff --git a/tests/tests_assemble_workflow/.test_bundle_rpm.py.swp b/tests/tests_assemble_workflow/.test_bundle_rpm.py.swp deleted file mode 100644 index b8e8655eb69b5d1ce616724f66240cd99fa72fa8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHNNpBoQ6fOc`3jsnd95@t{ps}Utwmpfh$SA8ZN`SUEN-S zfEzi$EfR3Z9d6vh4`4gwNDhc!0NjBa@KrA}(>>!whKNX1C!c4!x|Ubpt5@}2&ufie zy>OA9Fq;I|^Mu^LbFFp%-choAl#rX9WJRzIp&o5_Ieg7<8BH=4|toSJGLpE%CV zL>lW{#zxa5?-v!uf{^=?v&ilktwgwv5%QQvc+5O@Gw{b^w2;|fi%H47mdma6AX;s^ z!FpP5wG!V!VHA31xVfEl`k7`xGmtTGkQ{4HjH~ZQYA@5nFI>^YXa+O`ngPv#W zGy|Fe&46Y=Gw={Hz#Kwegk0BDz>nYmi}?Si=Lq=`xC69+Gr(zJ5_kpp>sdnn1nvQ! z0iOb&06yRW^S~T%7`S_o5DqxN0pQj%gnS6R1l&4+@4yN$4@?4wfRCRhSED#X}LniDs{oSkuX=Z z1&@}-m$Uz)q-ZI*+cs79Lln=ddL!)`nxy#>)M&M+VK}@kd`|0D9E6UD=(|~yC>CvI z$I=Q}+&v6iX?|@%Ej>wxO4P01?I5MEmsrx%j+wGP zl0FZay~?ihl-vd#!gxbTt*AFw0^w(kSV<=hTJKSc%oymx$aOKAcj+--w2=iaDiMs?oP^dj_u5jsgMlycU1s-dD>brcI zNmO|TmX%)$LFON=8F?&RTuB(X-p7iJcvRr8Ymmb3-C4Zy1fkWsskB?YZTOd)^!ai1W zSj+~KZ$WX%2{AW4w^lWoVQx_>l1jr}fRg%L8m#PrRar8~(s1>@H8wE+2RrL@hu>kk zd!J-}e)KG8fE4sysB3FqvD8Ybw7RpDe9LN=MgLnahpuZn=D?ZDmoUXOFfO|8Q?ijx z4}Y^LK}81jY^usJl{~%@riR;RTf}w|&(K9Ut>@D8ZMAL^U&P9dHC6EplYDd<@swKJ zv@isjowrtTPlXo-QCzl%tjR#OIGPrT;=N}1n$3slN}!^ZqEhB5r%o)%hb!t8_kl!g zO6|ZyErAU6G)|&2NFtk4SJ*1Y-9(MJBP3NuNL$@yQ08zN2jmD^U!H&Mn(NUQF@Q)T+n3}^;41DXNNfM!55pc&8%Xa+O`ngPv# aX5e9AK)tU~SNhJT|2O06OVH{MT7Lr@C1q;> From a905e3b7800a9888a6920c7d37f24020058dbc91 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 20 Apr 2022 18:44:22 +0000 Subject: [PATCH 6/8] Rename vars Signed-off-by: Peter Zhu --- src/test_workflow/integ_test/distribution.py | 10 ++++----- .../integ_test/distribution_rpm.py | 14 ++++++------ .../integ_test/distribution_tar.py | 12 +++++----- src/test_workflow/integ_test/service.py | 2 +- .../integ_test/service_opensearch.py | 12 +++++----- .../service_opensearch_dashboards.py | 12 +++++----- .../integ_test/test_distribution_rpm.py | 22 +++++++++---------- .../integ_test/test_distribution_tar.py | 18 +++++++-------- 8 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/test_workflow/integ_test/distribution.py b/src/test_workflow/integ_test/distribution.py index 195e83af65..e16664537f 100644 --- a/src/test_workflow/integ_test/distribution.py +++ b/src/test_workflow/integ_test/distribution.py @@ -18,7 +18,7 @@ def __init__(self, filename: str, version: str, work_dir: str) -> None: @property @abstractmethod - def get_install_dir(self) -> str: + def install_dir(self) -> str: """ Return the install directory for the distribution """ @@ -26,14 +26,14 @@ def get_install_dir(self) -> str: @property @abstractmethod - def get_config_dir(self, bundle_name: str) -> str: + def config_dir(self, bundle_name: str) -> str: """ Return the config directory for the distribution """ pass @abstractmethod - def install_distribution(self) -> None: + def install(self) -> None: """ The detailed method to install the distribution before start the service """ @@ -41,13 +41,13 @@ def install_distribution(self) -> None: @property @abstractmethod - def get_start_cmd(self) -> str: + def start_cmd(self) -> str: """ Return the start command for the distribution """ pass - def cleanup(self) -> None: + def uninstall(self) -> None: """ Allow distribution that is not 'tar' to do proper cleanup """ diff --git a/src/test_workflow/integ_test/distribution_rpm.py b/src/test_workflow/integ_test/distribution_rpm.py index db5361c252..85e26878e2 100644 --- a/src/test_workflow/integ_test/distribution_rpm.py +++ b/src/test_workflow/integ_test/distribution_rpm.py @@ -16,15 +16,15 @@ def __init__(self, filename: str, version: str, work_dir: str) -> None: super().__init__(filename, version, work_dir) @property - def get_install_dir(self) -> str: + def install_dir(self) -> str: return os.path.join(os.sep, "usr", "share", self.filename) @property - def get_config_dir(self) -> str: + def config_dir(self) -> str: return os.path.join(os.sep, "etc", self.filename) - def install_distribution(self, bundle_name: str) -> None: - logging.info(f"Installing {bundle_name} in {self.get_install_dir}") + def install(self, bundle_name: str) -> None: + logging.info(f"Installing {bundle_name} in {self.install_dir}") logging.info("rpm installation requires sudo, script will exit if current user does not have sudo access") rpm_install_cmd = " ".join( [ @@ -42,13 +42,13 @@ def install_distribution(self, bundle_name: str) -> None: subprocess.check_call(rpm_install_cmd, cwd=self.work_dir, shell=True) @property - def get_start_cmd(self) -> str: + def start_cmd(self) -> str: start_cmd_map = { "opensearch": "systemctl start opensearch", "opensearch-dashboards": "systemctl start opensearch-dashboards", } return start_cmd_map[self.filename] - def cleanup(self) -> None: - logging.info("Clean up packages after the test") + def uninstall(self) -> None: + logging.info("Uninstall {self.filename} package after the test") subprocess.check_call(f"yum remove -y '{self.filename}*'", shell=True) diff --git a/src/test_workflow/integ_test/distribution_tar.py b/src/test_workflow/integ_test/distribution_tar.py index 5c52b50866..b45c2dccc7 100644 --- a/src/test_workflow/integ_test/distribution_tar.py +++ b/src/test_workflow/integ_test/distribution_tar.py @@ -16,20 +16,20 @@ def __init__(self, filename: str, version: str, work_dir: str) -> None: super().__init__(filename, version, work_dir) @property - def get_install_dir(self) -> str: + def install_dir(self) -> str: return os.path.join(self.work_dir, f"{self.filename}-{self.version}") @property - def get_config_dir(self) -> str: - return os.path.join(self.get_install_dir, "config") + def config_dir(self) -> str: + return os.path.join(self.install_dir, "config") - def install_distribution(self, bundle_name: str) -> None: - logging.info(f"Installing {bundle_name} in {self.get_install_dir}") + def install(self, bundle_name: str) -> None: + logging.info(f"Installing {bundle_name} in {self.install_dir}") with tarfile.open(bundle_name, 'r:gz') as bundle_tar: bundle_tar.extractall(self.work_dir) @property - def get_start_cmd(self) -> str: + def start_cmd(self) -> str: start_cmd_map = { "opensearch": "./opensearch-tar-install.sh", "opensearch-dashboards": "./opensearch-dashboards", diff --git a/src/test_workflow/integ_test/service.py b/src/test_workflow/integ_test/service.py index 44c60f83f2..67587e0c24 100644 --- a/src/test_workflow/integ_test/service.py +++ b/src/test_workflow/integ_test/service.py @@ -45,7 +45,7 @@ def terminate(self): self.return_code = self.process_handler.terminate() - self.cleanup() + self.uninstall() return ServiceTerminationResult(self.return_code, self.process_handler.stdout_data, self.process_handler.stderr_data, self.log_files) diff --git a/src/test_workflow/integ_test/service_opensearch.py b/src/test_workflow/integ_test/service_opensearch.py index 6a6a5fb43f..a8a98ff38b 100644 --- a/src/test_workflow/integ_test/service_opensearch.py +++ b/src/test_workflow/integ_test/service_opensearch.py @@ -28,12 +28,12 @@ def __init__( self.dist = Distributions.get_distribution("opensearch", distribution, version, work_dir) self.dependency_installer = dependency_installer - self.install_dir = self.dist.get_install_dir + self.install_dir = self.dist.install_dir def start(self): - self.dist.install_distribution(self.download()) + self.dist.install(self.download()) - self.opensearch_yml_dir = os.path.join(self.dist.get_config_dir, "opensearch.yml") + self.opensearch_yml_dir = os.path.join(self.dist.config_dir, "opensearch.yml") self.security_plugin_dir = os.path.join(self.install_dir, "plugins", "opensearch-security") if not self.security_enabled and os.path.isdir(self.security_plugin_dir): @@ -42,11 +42,11 @@ def start(self): if self.additional_config: self.__add_plugin_specific_config(self.additional_config) - self.process_handler.start(self.dist.get_start_cmd, self.install_dir) + self.process_handler.start(self.dist.start_cmd, self.install_dir) logging.info(f"Started OpenSearch with parent PID {self.process_handler.pid}") - def cleanup(self): - self.dist.cleanup() + def uninstall(self): + self.dist.uninstall() def url(self, path=""): return f'{"https" if self.security_enabled else "http"}://{self.endpoint()}:{self.port()}{path}' diff --git a/src/test_workflow/integ_test/service_opensearch_dashboards.py b/src/test_workflow/integ_test/service_opensearch_dashboards.py index 2da51da30f..8304cd71e7 100644 --- a/src/test_workflow/integ_test/service_opensearch_dashboards.py +++ b/src/test_workflow/integ_test/service_opensearch_dashboards.py @@ -27,12 +27,12 @@ def __init__( ): super().__init__(work_dir, version, distribution, security_enabled, additional_config, dependency_installer) self.dist = Distributions.get_distribution("opensearch-dashboards", distribution, version, work_dir) - self.install_dir = self.dist.get_install_dir + self.install_dir = self.dist.install_dir def start(self): - self.dist.install_distribution(self.download()) + self.dist.install(self.download()) - self.opensearch_dashboards_yml_dir = os.path.join(self.dist.get_config_dir, "opensearch_dashboards.yml") + self.opensearch_dashboards_yml_dir = os.path.join(self.dist.config_dir, "opensearch_dashboards.yml") self.executable_dir = os.path.join(self.install_dir, "bin") if not self.security_enabled: @@ -43,11 +43,11 @@ def start(self): if self.additional_config: self.__add_plugin_specific_config(self.additional_config) - self.process_handler.start(self.dist.get_start_cmd, self.executable_dir) + self.process_handler.start(self.dist.start_cmd, self.executable_dir) logging.info(f"Started OpenSearch Dashboards with parent PID {self.process_handler.pid}") - def cleanup(self): - self.dist.cleanup() + def uninstall(self): + self.dist.uninstall() def __set_logging_dest(self): self.log_dir = os.path.join(self.install_dir, "logs") diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py index 6c0a5025d8..17f6ae61ea 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py @@ -24,27 +24,27 @@ def test_distribution_rpm_vars(self) -> None: self.assertEqual(self.distribution_rpm.version, '1.3.0') self.assertEqual(self.distribution_rpm.work_dir, self.work_dir) - def test_get_install_dir(self) -> None: - self.assertEqual(self.distribution_rpm.get_install_dir, os.path.join(os.sep, "usr", "share", "opensearch")) + def test_install_dir(self) -> None: + self.assertEqual(self.distribution_rpm.install_dir, os.path.join(os.sep, "usr", "share", "opensearch")) - def test_get_config_dir(self) -> None: - self.assertEqual(self.distribution_rpm.get_config_dir, os.path.join(os.sep, "etc", "opensearch")) + def test_config_dir(self) -> None: + self.assertEqual(self.distribution_rpm.config_dir, os.path.join(os.sep, "etc", "opensearch")) @patch("subprocess.check_call") - def test_install_distribution(self, check_call_mock: Mock) -> None: - self.distribution_rpm.install_distribution("opensearch.rpm") + def test_install(self, check_call_mock: Mock) -> None: + self.distribution_rpm.install("opensearch.rpm") args_list = check_call_mock.call_args_list self.assertEqual(check_call_mock.call_count, 1) self.assertEqual("yum remove -y opensearch && yum install -y opensearch.rpm", args_list[0][0][0]) - def test_get_srpmt_cmd(self) -> None: - self.assertEqual(self.distribution_rpm.get_start_cmd, "systemctl start opensearch") - self.assertEqual(self.distribution_rpm_dashboards.get_start_cmd, "systemctl start opensearch-dashboards") + def test_start_cmd(self) -> None: + self.assertEqual(self.distribution_rpm.start_cmd, "systemctl start opensearch") + self.assertEqual(self.distribution_rpm_dashboards.start_cmd, "systemctl start opensearch-dashboards") @patch("subprocess.check_call") - def test_cleanup(self, check_call_mock: Mock) -> None: - self.distribution_rpm.cleanup() + def test_uninstall(self, check_call_mock: Mock) -> None: + self.distribution_rpm.uninstall() args_list = check_call_mock.call_args_list self.assertEqual(check_call_mock.call_count, 1) diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py index f4601a3d4d..0805c9edc9 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_tar.py @@ -24,22 +24,22 @@ def test_distribution_tar_vars(self) -> None: self.assertEqual(self.distribution_tar.version, '1.3.0') self.assertEqual(self.distribution_tar.work_dir, self.work_dir) - def test_get_install_dir(self) -> None: - self.assertEqual(self.distribution_tar.get_install_dir, os.path.join(self.work_dir, "opensearch-1.3.0")) + def test_install_dir(self) -> None: + self.assertEqual(self.distribution_tar.install_dir, os.path.join(self.work_dir, "opensearch-1.3.0")) - def test_get_config_dir(self) -> None: - self.assertEqual(self.distribution_tar.get_config_dir, os.path.join(self.work_dir, "opensearch-1.3.0", "config")) + def test_config_dir(self) -> None: + self.assertEqual(self.distribution_tar.config_dir, os.path.join(self.work_dir, "opensearch-1.3.0", "config")) - def test_install_distribution(self) -> None: + def test_install(self) -> None: with patch("tarfile.open") as mock_tarfile_open: mock_tarfile_extractall = MagicMock() mock_tarfile_open.return_value.__enter__.return_value.extractall = mock_tarfile_extractall - self.distribution_tar.install_distribution(os.path.join(self.work_dir, "artifacts", "dist", "opensearch-min-1.3.0-linux-x64.tar.gz")) + self.distribution_tar.install(os.path.join(self.work_dir, "artifacts", "dist", "opensearch-min-1.3.0-linux-x64.tar.gz")) mock_tarfile_open.assert_called_with(os.path.join(self.work_dir, "artifacts", "dist", "opensearch-min-1.3.0-linux-x64.tar.gz"), "r:gz") mock_tarfile_extractall.assert_called_with(self.work_dir) - def test_get_start_cmd(self) -> None: - self.assertEqual(self.distribution_tar.get_start_cmd, "./opensearch-tar-install.sh") - self.assertEqual(self.distribution_tar_dashboards.get_start_cmd, "./opensearch-dashboards") + def test_start_cmd(self) -> None: + self.assertEqual(self.distribution_tar.start_cmd, "./opensearch-tar-install.sh") + self.assertEqual(self.distribution_tar_dashboards.start_cmd, "./opensearch-dashboards") From 8ed09387ab0f00e119ad4e7b39e431c4c033cb2b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 20 Apr 2022 18:50:07 +0000 Subject: [PATCH 7/8] More map removal Signed-off-by: Peter Zhu --- src/test_workflow/integ_test/distribution_rpm.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/test_workflow/integ_test/distribution_rpm.py b/src/test_workflow/integ_test/distribution_rpm.py index 85e26878e2..a216eadaf0 100644 --- a/src/test_workflow/integ_test/distribution_rpm.py +++ b/src/test_workflow/integ_test/distribution_rpm.py @@ -43,11 +43,7 @@ def install(self, bundle_name: str) -> None: @property def start_cmd(self) -> str: - start_cmd_map = { - "opensearch": "systemctl start opensearch", - "opensearch-dashboards": "systemctl start opensearch-dashboards", - } - return start_cmd_map[self.filename] + return f"systemctl start {self.filename}" def uninstall(self) -> None: logging.info("Uninstall {self.filename} package after the test") From 3e17ea3c051b6572be786df9bfb384bee1280b05 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 20 Apr 2022 18:52:46 +0000 Subject: [PATCH 8/8] More tweaks Signed-off-by: Peter Zhu --- src/test_workflow/integ_test/distribution_rpm.py | 2 +- .../test_integ_workflow/integ_test/test_distribution_rpm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_workflow/integ_test/distribution_rpm.py b/src/test_workflow/integ_test/distribution_rpm.py index a216eadaf0..3a71ddbfce 100644 --- a/src/test_workflow/integ_test/distribution_rpm.py +++ b/src/test_workflow/integ_test/distribution_rpm.py @@ -47,4 +47,4 @@ def start_cmd(self) -> str: def uninstall(self) -> None: logging.info("Uninstall {self.filename} package after the test") - subprocess.check_call(f"yum remove -y '{self.filename}*'", shell=True) + subprocess.check_call(f"yum remove -y {self.filename}", shell=True) diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py index 17f6ae61ea..4a32dbc4fa 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_rpm.py @@ -48,4 +48,4 @@ def test_uninstall(self, check_call_mock: Mock) -> None: args_list = check_call_mock.call_args_list self.assertEqual(check_call_mock.call_count, 1) - self.assertEqual("yum remove -y 'opensearch*'", args_list[0][0][0]) + self.assertEqual("yum remove -y opensearch", args_list[0][0][0])