From 95f99ea4945ffc336ebd2e9480294090dcbdbf4e Mon Sep 17 00:00:00 2001 From: Divya Madala <113469545+Divyaasm@users.noreply.github.com> Date: Thu, 9 May 2024 14:29:22 -0700 Subject: [PATCH] [Benchmark] Add mechanism to retreive the logs from test-execution.json (#4686) Signed-off-by: Divya Madala --- Pipfile | 1 + Pipfile.lock | 132 ++++++-- .../benchmark-test-endpoint.jenkinsfile | 5 + .../benchmark_test/benchmark_test_suite.py | 35 +- .../TestRunBenchmarkTestEndpoint.groovy | 9 +- ...ark-test-endpoint-insecure.jenkinsfile.txt | 7 +- ...hmark-test-endpoint-secure.jenkinsfile.txt | 7 +- .../test_benchmark_test_suite.py | 305 ++++++++++++------ 8 files changed, 368 insertions(+), 133 deletions(-) diff --git a/Pipfile b/Pipfile index a311584ba4..07b57cb0a8 100644 --- a/Pipfile +++ b/Pipfile @@ -41,6 +41,7 @@ ruamel-yaml = "~=0.17.21" markdownify = "~=0.12.1" mistune = "~=3.0.1" semver = ">=3,<4" +pandas = "~=2.2.2" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index a1ab9acdee..014efa95a2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f7dbe7be62a029687f5e25390043bbc05e0ee46378321fa4f137bf129e790ae6" + "sha256": "c3d3fb11cde134753acb475da5f9f66e77c5dbaa4dd4116ddb3c336190c8528d" }, "pipfile-spec": 6, "requires": { @@ -159,11 +159,11 @@ }, "filelock": { "hashes": [ - "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f", - "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4" + "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f", + "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a" ], "markers": "python_version >= '3.8'", - "version": "==3.13.4" + "version": "==3.14.0" }, "flake8": { "hashes": [ @@ -176,11 +176,11 @@ }, "identify": { "hashes": [ - "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791", - "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e" + "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa", + "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d" ], "markers": "python_version >= '3.8'", - "version": "==2.5.35" + "version": "==2.5.36" }, "idna": { "hashes": [ @@ -302,6 +302,48 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", "version": "==1.8.0" }, + "numpy": { + "hashes": [ + "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", + "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", + "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", + "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", + "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", + "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a", + "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea", + "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c", + "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", + "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", + "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be", + "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", + "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a", + "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", + "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", + "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd", + "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c", + "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", + "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0", + "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c", + "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", + "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", + "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0", + "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6", + "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2", + "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", + "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30", + "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", + "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5", + "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07", + "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", + "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4", + "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764", + "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", + "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3", + "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f" + ], + "markers": "python_version < '3.11'", + "version": "==1.26.4" + }, "packaging": { "hashes": [ "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", @@ -310,6 +352,42 @@ "markers": "python_version >= '3.7'", "version": "==24.0" }, + "pandas": { + "hashes": [ + "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863", + "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2", + "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1", + "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad", + "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db", + "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76", + "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51", + "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32", + "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08", + "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b", + "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4", + "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921", + "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288", + "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee", + "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0", + "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24", + "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99", + "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151", + "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd", + "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce", + "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57", + "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef", + "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54", + "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a", + "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238", + "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23", + "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772", + "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce", + "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==2.2.2" + }, "pathspec": { "hashes": [ "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", @@ -328,19 +406,19 @@ }, "platformdirs": { "hashes": [ - "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068", - "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768" + "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf", + "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1" ], "markers": "python_version >= '3.8'", - "version": "==4.2.0" + "version": "==4.2.1" }, "pluggy": { "hashes": [ - "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981", - "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be" + "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", + "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669" ], "markers": "python_version >= '3.8'", - "version": "==1.4.0" + "version": "==1.5.0" }, "pre-commit": { "hashes": [ @@ -430,6 +508,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0.post0" }, "pytz": { @@ -598,11 +677,11 @@ }, "setuptools": { "hashes": [ - "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e", - "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c" + "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987", + "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32" ], "markers": "python_version >= '3.8'", - "version": "==69.2.0" + "version": "==69.5.1" }, "six": { "hashes": [ @@ -638,11 +717,11 @@ }, "tcolorpy": { "hashes": [ - "sha256:d0926480aa5012f34877d69fc3b670f207dc165674e68ad07458fa6ee5b12724", - "sha256:f0dceb1cb95e554cee63024b3cd2fd8d4628c568773de2d1e6b4f0478461901c" + "sha256:8c15cb3167f30b0a433d72297e9d68667c825bd9e2af41c8dd7dfbd3d7f7e207", + "sha256:8cea0bf5f8cf03f77528a9acfbf312df935573892ba5ea3b2516e61fa54de9a5" ], "markers": "python_version >= '3.7'", - "version": "==0.1.4" + "version": "==0.1.6" }, "toml": { "hashes": [ @@ -724,6 +803,7 @@ "sha256:a9e0f0f88dc835739b0c1ca51ee90d04ca2a897a71af79de9aec5f38cb0a5342", "sha256:b845b06a1c7e54b8e5b4c683043de0d9caf205e7434b3edc678ff2411979b8f6" ], + "index": "pypi", "markers": "python_version >= '3.8'", "version": "==6.0.12.20240311" }, @@ -751,6 +831,14 @@ "markers": "python_version >= '3.8'", "version": "==4.11.0" }, + "tzdata": { + "hashes": [ + "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd", + "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252" + ], + "markers": "python_version >= '2'", + "version": "==2024.1" + }, "urllib3": { "hashes": [ "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07", @@ -770,11 +858,11 @@ }, "virtualenv": { "hashes": [ - "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a", - "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197" + "sha256:604bfdceaeece392802e6ae48e69cec49168b9c5f4a44e483963f9242eb0e78b", + "sha256:7aa9982a728ae5892558bff6a2839c00b9ed145523ece2274fad6f414690ae75" ], "markers": "python_version >= '3.7'", - "version": "==20.25.1" + "version": "==20.26.1" }, "yamlfix": { "hashes": [ diff --git a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile index 2ef01e3781..9631f0acda 100644 --- a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile +++ b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile @@ -121,6 +121,7 @@ parameters { password: PASSWORD, workload: TEST_WORKLOAD, userTag: USER_TAGS.isEmpty() ? "security-enabled:${SECURITY_ENABLED}" : "${USER_TAGS},security-enabled:${SECURITY_ENABLED}", + suffix: "${BUILD_NUMBER}", workloadParams: WORKLOAD_PARAMS, testProcedure: TEST_PROCEDURE, excludeTasks: EXCLUDE_TASKS, @@ -129,10 +130,14 @@ parameters { captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT, telemetryParams: TELEMETRY_PARAMS ) + stash includes: 'test_execution*.csv', name: "benchmark" + } } post { always { + unstash "benchmark" + archiveArtifacts artifacts: 'test_execution*.csv' postCleanup() } } diff --git a/src/test_workflow/benchmark_test/benchmark_test_suite.py b/src/test_workflow/benchmark_test/benchmark_test_suite.py index 4dd6a4476e..7b95d52e33 100644 --- a/src/test_workflow/benchmark_test/benchmark_test_suite.py +++ b/src/test_workflow/benchmark_test/benchmark_test_suite.py @@ -5,11 +5,17 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. +import glob +import json import logging import os +import shutil import subprocess from typing import Any +import pandas as pd + +from system.temporary_directory import TemporaryDirectory from test_workflow.benchmark_test.benchmark_args import BenchmarkArgs @@ -37,7 +43,7 @@ def __init__( self.password = password # Pass the cluster endpoints with -t for multi-cluster use cases(e.g. cross-cluster-replication) - self.command = 'docker run --rm' + self.command = f'docker run --name docker-container-{self.args.stack_suffix}' if self.args.benchmark_config: self.command += f" -v {args.benchmark_config}:/opensearch-benchmark/.benchmark/benchmark.ini" self.command += f" opensearchproject/opensearch-benchmark:latest execute-test --workload={self.args.workload} " \ @@ -67,11 +73,34 @@ def __init__( if self.args.telemetry_params: self.command += f" --telemetry-params '{self.args.telemetry_params}'" - def execute(self) -> None: if self.security: self.command += f' --client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'{self.args.username}\',basic_auth_password:\'{self.password}\'"' else: self.command += ' --client-options="timeout:300"' + + def execute(self) -> None: log_info = f"Executing {self.command.replace(self.endpoint, len(self.endpoint) * '*').replace(self.args.username, len(self.args.username) * '*')}" logging.info(log_info.replace(self.password, len(self.password) * '*') if self.password else log_info) - subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True) + try: + subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True) + if self.args.cluster_endpoint: + self.convert() + finally: + self.cleanup() + + def convert(self) -> None: + with TemporaryDirectory() as work_dir: + subprocess.check_call(f"docker cp docker-container-{self.args.stack_suffix}:opensearch-benchmark/. {str(work_dir.path)}", cwd=os.getcwd(), shell=True) + file_path = glob.glob(os.path.join(str(work_dir.path), "test_executions", "*", "test_execution.json")) + with open(file_path[0]) as file: + data = json.load(file) + formatted_data = pd.json_normalize(data["results"]["op_metrics"]) + formatted_data.to_csv(os.path.join(os.getcwd(), f"test_execution_{self.args.stack_suffix}.csv"), index=False) + df = pd.read_csv(os.path.join(os.getcwd(), f"test_execution_{self.args.stack_suffix}.csv")) + pd.set_option('display.width', int(2 * shutil.get_terminal_size().columns)) + pd.set_option('display.max_rows', None) + pd.set_option('display.max_columns', None) + logging.info(f"\n{df}") + + def cleanup(self) -> None: + subprocess.check_call(f"docker rm docker-container-{self.args.stack_suffix}", cwd=os.getcwd(), shell=True) diff --git a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy index 77923dce0b..52271c41a0 100644 --- a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy +++ b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy @@ -36,6 +36,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ helper.registerAllowedMethod("s3Download", [Map]) helper.registerAllowedMethod("uploadTestResults", [Map]) helper.registerAllowedMethod("s3Upload", [Map]) + helper.registerAllowedMethod('unstash', [String.class], null) helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> @@ -56,6 +57,9 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ binding.setVariable('GITHUB_TOKEN', 'test_token') binding.setVariable('USER_TAGS', 'run-type:test') binding.setVariable('WORKLOAD_PARAMS', '') + binding.setVariable('env', ['BUILD_NUMBER': '307']) + binding.setVariable('BUILD_NUMBER', '307') + binding.setVariable('SUFFIX', '1234') binding.setVariable('TEST_PROCEDURE', 'append-no-conflicts') binding.setVariable('EXCLUDE_TASKS', '') binding.setVariable('INCLUDE_TASKS', '') @@ -98,6 +102,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ @Test void testRunSecureBenchmarkTestScript_verifyScriptExecutions() { addParam('SECURITY_ENABLED', true) + runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { @@ -106,7 +111,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ assertThat(testScriptCommands.size(), equalTo(1)) assertThat(testScriptCommands, hasItems( - "set +x && ./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "set +x && ./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --suffix 307 --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'" )) } @Test @@ -127,7 +132,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ assertThat(testScriptCommands.size(), equalTo(1)) assertThat(testScriptCommands, hasItems( - "set +x && ./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "set +x && ./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --suffix 307 --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'" )) } diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt index aa2611ae3e..79874e3899 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt @@ -15,7 +15,7 @@ benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) benchmark-test-endpoint.script(groovy.lang.Closure) benchmark-test-endpoint.echo(security-enabled: true) - benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, username=, password=, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, username=, password=, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, suffix=307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@6.4.3, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -28,6 +28,9 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --suffix 307 --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + benchmark-test-endpoint.stash({includes=test_execution*.csv, name=benchmark}) + benchmark-test-endpoint.unstash(benchmark) + benchmark-test-endpoint.archiveArtifacts({artifacts=test_execution*.csv}) benchmark-test-endpoint.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt index aa2611ae3e..79874e3899 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt @@ -15,7 +15,7 @@ benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) benchmark-test-endpoint.script(groovy.lang.Closure) benchmark-test-endpoint.echo(security-enabled: true) - benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, username=, password=, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, username=, password=, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, suffix=307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@6.4.3, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -28,6 +28,9 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --suffix 307 --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + benchmark-test-endpoint.stash({includes=test_execution*.csv, name=benchmark}) + benchmark-test-endpoint.unstash(benchmark) + benchmark-test-endpoint.archiveArtifacts({artifacts=test_execution*.csv}) benchmark-test-endpoint.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py index a171a4adcb..6f6c7bbfd7 100644 --- a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py +++ b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py @@ -5,9 +5,11 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. +import os +import tempfile import unittest from typing import Any -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch from test_workflow.benchmark_test.benchmark_test_suite import BenchmarkTestSuite from test_workflow.integ_test.utils import get_password @@ -17,12 +19,14 @@ class TestBenchmarkTestSuite(unittest.TestCase): def setUp(self, **kwargs: Any) -> None: with patch('test_workflow.integ_test.utils.get_password') as mock_get_password: self.args = Mock() + self.args.insecure = True self.args.workload = "nyc_taxis" self.args.version = '2.9.0' self.args.benchmark_config = kwargs['config'] if 'config' in kwargs else None mock_get_password.return_value = get_password('2.11.0') self.args.username = "admin" self.password = "myStrongPassword123!" + self.args.cluster_endpoint = None self.args.user_tag = kwargs['tags'] if 'tags' in kwargs else None self.args.workload_params = kwargs['workload_params'] if 'workload_params' in kwargs else None self.args.telemetry = kwargs['telemetry'] if 'telemetry' in kwargs else None @@ -31,118 +35,215 @@ def setUp(self, **kwargs: Any) -> None: self.args.exclude_tasks = kwargs['exclude_tasks'] if 'exclude_tasks' in kwargs else None self.args.include_tasks = kwargs['include_tasks'] if 'include_tasks' in kwargs else None self.endpoint = "abc.com" - self.benchmark_test_suite = BenchmarkTestSuite(endpoint=self.endpoint, security=False, args=self.args, password=self.password) - def test_execute_default(self) -> None: - with patch("subprocess.check_call") as mock_check_call: - self.benchmark_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 1) - self.assertEqual(self.benchmark_test_suite.command, - 'docker run --rm opensearchproject/opensearch-benchmark:latest execute-test --workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com --client-options="timeout:300"') - - def test_execute_security_enabled_version_212_or_greater(self) -> None: - benchmark_test_suite = BenchmarkTestSuite(endpoint=self.endpoint, security=True, args=self.args, password=self.password) - with patch("subprocess.check_call") as mock_check_call: - benchmark_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 1) - self.assertEqual(benchmark_test_suite.command, - 'docker run --rm opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis --pipeline=benchmark-only ' - '--target-hosts=abc.com --client-options="timeout:300,use_ssl:true,' - 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'myStrongPassword123!\'"') - - def test_execute_security_enabled(self) -> None: - benchmark_test_suite = BenchmarkTestSuite(endpoint=self.endpoint, security=True, args=self.args, password="admin") - with patch("subprocess.check_call") as mock_check_call: - benchmark_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 1) - self.assertEqual(benchmark_test_suite.command, - 'docker run --rm opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis --pipeline=benchmark-only ' - '--target-hosts=abc.com --client-options="timeout:300,use_ssl:true,' - 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') - - def test_execute_default_with_optional_args(self) -> None: + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + def test_execute_default(self, mock_check_call: Mock) -> None: + + self.args.insecure = True + mock_check_call.return_value = 0 + mock_convert = MagicMock() + with patch.object(BenchmarkTestSuite, 'convert', mock_convert): + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test ' + f'--workload=nyc_taxis --pipeline=benchmark-only --target-hosts=abc.com:80 --client-options="timeout:300"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_security_enabled_version_212_or_greater(self, mock_convert: Mock, mock_check_call: Mock) -> None: + mock_check_call.return_value = 0 + self.args.insecure = False + test_suite = BenchmarkTestSuite("abc.com:443", True, self.args, self.password) + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test' + f' --workload=nyc_taxis --pipeline=benchmark-only ' + f'--target-hosts=abc.com:443 ' + f'--client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'myStrongPassword123!\'"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_security_enabled(self, mock_convert: Mock, mock_check_call: Mock) -> None: + mock_check_call.return_value = 0 + self.args.insecure = True + test_suite = BenchmarkTestSuite("abc.com:443", True, self.args, "admin") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis --pipeline=benchmark-only ' + '--target-hosts=abc.com:443 --client-options="timeout:300,use_ssl:true,' + 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_default_with_optional_args(self, mock_convert: Mock, mock_check_call: Mock) -> None: + mock_check_call.return_value = 0 + self.args.telemetry = [] + self.args.telemetry.append('node-stats') TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", - workload_params="{\"number_of_replicas\":\"1\"}", telemetry=['node-stats', 'test'], + workload_params="{\"number_of_replicas\":\"1\"}", telemetry=['node-stats'], telemetry_params="{\"example_key\":\"example_value\"}") - with patch("subprocess.check_call") as mock_check_call: - self.benchmark_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 1) - self.assertEqual(self.benchmark_test_suite.command, 'docker run --rm -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--user-tag="key1:value1,key2:value2" --telemetry node-stats,test, --telemetry-params \'{"example_key":"example_value"}\' ' - '--client-options="timeout:300"') - - def test_execute_default_with_no_telemetry_params(self) -> None: + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--user-tag="key1:value1,key2:value2" --telemetry node-stats, --telemetry-params \'{"example_key":"example_value"}\' ' + '--client-options="timeout:300"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_default_with_no_telemetry_params(self, mock_convert: Mock, mock_check_call: Mock) -> None: + mock_check_call.return_value = 0 TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", telemetry=['node-stats', 'test']) - with patch("subprocess.check_call") as mock_check_call: - self.benchmark_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 1) - self.assertEqual(self.benchmark_test_suite.command, 'docker run --rm -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--user-tag="key1:value1,key2:value2" --telemetry node-stats,test, ' - '--client-options="timeout:300"') - - def test_execute_with_test_procedure_params(self) -> None: + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--user-tag="key1:value1,key2:value2" --telemetry node-stats,test, ' + '--client-options="timeout:300"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_with_test_procedure_params(self, mock_convert: Mock, mock_check_call: Mock) -> None: + mock_check_call.return_value = 0 + self.args.insecure = True TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", test_procedure="test-proc1,test-proc2") - with patch("subprocess.check_call") as mock_check_call: - self.benchmark_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 1) - self.assertEqual(self.benchmark_test_suite.command, 'docker run --rm -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--test-procedure="test-proc1,test-proc2" ' - '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') - - def test_execute_with_include_exclude_params(self) -> None: + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--test-procedure="test-proc1,test-proc2" ' + '--user-tag="key1:value1,key2:value2" ' + '--client-options="timeout:300"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_with_include_exclude_params(self, mock_convert: Mock, mock_check_call: Mock) -> None: + mock_check_call.return_value = 0 + self.args.insecure = True TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", include_tasks="task1,type:index", exclude_tasks="task2,type:search") - with patch("subprocess.check_call") as mock_check_call: - self.benchmark_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 1) - self.assertEqual(self.benchmark_test_suite.command, 'docker run --rm -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--exclude-tasks="task2,type:search" ' - '--include-tasks="task1,type:index" ' - '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') - - def test_execute_with_all_benchmark_optional_params(self) -> None: + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--exclude-tasks="task2,type:search" ' + '--include-tasks="task1,type:index" ' + '--user-tag="key1:value1,key2:value2" ' + '--client-options="timeout:300"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_with_all_benchmark_optional_params(self, mock_convert: Mock) -> None: + self.args.insecure = True TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", test_procedure="test-proc1,test-proc2", include_tasks="task1,type:index", exclude_tasks="task2,type:search") with patch("subprocess.check_call") as mock_check_call: - self.benchmark_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 1) - self.assertEqual(self.benchmark_test_suite.command, 'docker run --rm -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--test-procedure="test-proc1,test-proc2" ' - '--exclude-tasks="task2,type:search" ' - '--include-tasks="task1,type:index" ' - '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--test-procedure="test-proc1,test-proc2" ' + '--exclude-tasks="task2,type:search" ' + '--include-tasks="task1,type:index" ' + '--user-tag="key1:value1,key2:value2" ' + '--client-options="timeout:300"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_cluster_endpoint(self, mock_convert: Mock, mock_check_call: Mock) -> None: + mock_check_call.return_value = 0 + self.args.cluster_endpoint = "abc.com" + self.args.insecure = True + test_suite = BenchmarkTestSuite("abc.com:443", True, self.args, "admin") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + self.assertEqual(mock_convert.call_count, 1) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis --pipeline=benchmark-only ' + '--target-hosts=abc.com:443 --client-options="timeout:300,use_ssl:true,' + 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') + + @patch('pandas.json_normalize') + @patch('pandas.read_csv') + @patch('json.load') + @patch('builtins.open') + @patch('logging.info') + @patch('shutil.get_terminal_size') + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + def test_convert(self, mock_check_call: Mock, mock_get_terminal_size: Mock, mock_logging_info: Mock, mock_open: Mock, mock_json_load: Mock, mock_read_csv: Mock, + mock_json_normalize: Mock) -> None: + self.args.cluster_endpoint = "abc.com" + mock_get_terminal_size.return_value = MagicMock(columns=80) + mock_open.return_value = MagicMock() + mock_json_load.return_value = {"results": {"op_metrics": [{"metric": "value"}]}} + mock_json_normalize.return_value = MagicMock() + mock_read_csv.return_value = MagicMock() + + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: + mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' + with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: + mock_glob.return_value = ['/mock/test_execution.json'] + test_suite.convert() + mock_temp_directory.assert_called_once() + mock_check_call.assert_called_with(f"docker cp docker-container-{test_suite.args.stack_suffix}:opensearch-benchmark/. /mock/temp/dir", cwd=os.getcwd(), shell=True) + mock_open.assert_called_once_with("/mock/test_execution.json") + mock_json_load.assert_called_once() + mock_json_normalize.assert_called_once() + mock_read_csv.assert_called_once() + mock_logging_info.assert_called()