Added groovy script for running benchmark tests (#205)

Signed-off-by: Rishabh Singh <[email protected]>
(cherry picked from commit 762ba18)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

tests/jenkins/TestRunBenchmarkTestScript.groovy

@@ -0,0 +1,104 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.hamcrest.CoreMatchers.equalTo
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.MatcherAssert.assertThat
+
+class TestRunBenchmarkTestScript extends BuildPipelineTest {
+
+    @Before
+    void setUp() {
+        this.registerLibTester(new RunBenchmarkTestScriptLibTester(
+                'tests/data/opensearch-1.3.0-bundle.yml',
+                'true',
+                'nyc_taxis',
+                'true',
+                'false',
+                'true',
+                '',
+                '',
+                '',
+                'key2:value2',
+                'cluster.indices.replication.strategy:SEGMENT',
+                'false'
+        ))
+        super.setUp()
+    }
+
+    @Test
+    public void testRunBenchmarkTestScript_PipelineSingleNode() {
+        super.testPipeline("tests/jenkins/jobs/BenchmarkTest_Jenkinsfile")
+    }
+
+    @Test
+    void testRunBenchmarkTestScript_verifyArtifactDownloads() {
+        runScript("tests/jenkins/jobs/BenchmarkTest_Jenkinsfile")
+
+        def curlCommands = getCommandExecutions('sh', 'curl').findAll {
+            shCommand -> shCommand.contains('curl')
+        }
+
+        assertThat(curlCommands.size(), equalTo(2))
+        assertThat(curlCommands, hasItem(
+                "curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString()
+        ))
+
+        def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll {
+            shCommand -> shCommand.contains('bucket')
+        }
+
+        assertThat(s3DownloadCommands.size(), equalTo(4))
+        assertThat(s3DownloadCommands, hasItem(
+                "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}".toString()
+        ))
+        assertThat(s3DownloadCommands, hasItem(
+                "{file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}".toString()
+        ))
+    }
+
+    @Test
+    void testRunPerfTestScript_verifyScriptExecutionsSingleNode() {
+        runScript("tests/jenkins/jobs/BenchmarkTest_Jenkinsfile")
+
+        def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
+            shCommand -> shCommand.contains('./test.sh')
+        }
+
+        assertThat(testScriptCommands.size(), equalTo(2))
+        assertThat(testScriptCommands, hasItem(
+                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true  --single-node  --use-50-percent-heap  --suffix 307-secure      --workload-params key2:value2 --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString()
+        ))
+        assertThat(testScriptCommands, hasItem(
+		        "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node  --use-50-percent-heap  --suffix 307      --workload-params key2:value2 --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString()
+        ))
+    }
+
+    def getCommandExecutions(methodName, command) {
+        def shCommands = helper.callStack.findAll {
+            call ->
+                call.methodName == methodName
+        }.
+                collect {
+                    call ->
+                        callArgsToString(call)
+                }.findAll {
+            shCommand ->
+                shCommand.contains(command)
+        }
+
+        return shCommands
+    }
+
+}

tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy

@@ -0,0 +1,79 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.hamcrest.CoreMatchers.equalTo
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.MatcherAssert.assertThat
+
+class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest {
+
+    @Before
+    void setUp() {
+        this.registerLibTester(new RunBenchmarkTestScriptLibTester(
+                'tests/data/opensearch-1.3.0-bundle.yml',
+                'true',
+                'nyc_taxis',
+                'false',
+                'false',
+                'true',
+                '3',
+                '3',
+                'key1:value1',
+                'key2:value2',
+                'cluster.indices.replication.strategy:SEGMENT',
+                'true'
+        ))
+        super.setUp()
+    }
+
+    @Test
+    public void testRunBenchmarkTestScript_PipelineMultiNode() {
+        super.testPipeline("tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile")
+    }
+
+
+    @Test
+    void testRunPerfTestScript_verifyScriptExecutionsMultiNode() {
+        runScript("tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile")
+
+        def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
+            shCommand -> shCommand.contains('./test.sh')
+        }
+
+        assertThat(testScriptCommands.size(), equalTo(2))
+        assertThat(testScriptCommands, hasItem(
+                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true    --use-50-percent-heap --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3    --workload-params key2:value2 --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString()
+        ))
+        assertThat(testScriptCommands, hasItem(
+                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security   --use-50-percent-heap --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3    --workload-params key2:value2 --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString()
+        ))
+    }
+
+    def getCommandExecutions(methodName, command) {
+        def shCommands = helper.callStack.findAll {
+            call ->
+                call.methodName == methodName
+        }.
+                collect {
+                    call ->
+                        callArgsToString(call)
+                }.findAll {
+            shCommand ->
+                shCommand.contains(command)
+        }
+
+        return shCommands
+    }
+
+}

tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile

@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('benchmark-test') {
+            parallel {
+                stage('test-with-security') {
+                    steps {
+                        script {
+                            def bundleManifestObj = downloadBuildManifest(
+                                url: BUNDLE_MANIFEST_URL,
+                                path: BUNDLE_MANIFEST
+                            )
+                            runBenchmarkTestScript(bundleManifest: BUNDLE_MANIFEST,
+                            workload: TEST_WORKLOAD,
+                            insecure: "false",
+                            singleNode: SINGLE_NODE_CLUSTER,
+                            minDistribution: MIN_DISTRIBUTION,
+                            use50PercentHeap: USE_50_PERCENT_HEAP,
+                            suffix: "${env.BUILD_NUMBER}-secure",
+                            managerNodeCount: MANAGER_NODE_COUNT,
+                            dataNodeCount: DATA_NODE_COUNT,
+                            clientNodeCount: CLIENT_NODE_COUNT,
+                            ingestNodeCount: INGEST_NODE_COUNT,
+                            mlNodeCount: ML_NODE_COUNT,
+                            userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true",
+                            workloadParams: WORKLOAD_PARAMS,
+                            additionalConfig: ADDITIONAL_CONFIG,
+                            dataStorageSize: DATA_NODE_STORAGE,
+                            mlStorageSize: ML_NODE_STORAGE,
+                            jvmSysProps: JVM_SYS_PROPS,
+                            captureNodeStat: CAPTURE_NODE_STAT
+                            )
+                        }
+                    }
+                }
+                stage('test-without-security') {
+                    steps {
+                        script {
+                            def bundleManifestObj = downloadBuildManifest(
+                                url: BUNDLE_MANIFEST_URL,
+                                path: BUNDLE_MANIFEST
+                            )
+                            runBenchmarkTestScript(bundleManifest: BUNDLE_MANIFEST,
+                            workload: TEST_WORKLOAD,
+                            insecure: "true",
+                            singleNode: SINGLE_NODE_CLUSTER,
+                            minDistribution: MIN_DISTRIBUTION,
+                            use50PercentHeap: USE_50_PERCENT_HEAP,
+                            suffix: "${env.BUILD_NUMBER}",
+                            managerNodeCount: MANAGER_NODE_COUNT,
+                            dataNodeCount: DATA_NODE_COUNT,
+                            clientNodeCount: CLIENT_NODE_COUNT,
+                            ingestNodeCount: INGEST_NODE_COUNT,
+                            mlNodeCount: ML_NODE_COUNT,
+                            userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false",
+                            workloadParams: WORKLOAD_PARAMS,
+                            additionalConfig: ADDITIONAL_CONFIG,
+                            dataStorageSize: DATA_NODE_STORAGE,
+                            mlStorageSize: ML_NODE_STORAGE,
+                            jvmSysProps: JVM_SYS_PROPS,
+                            captureNodeStat: CAPTURE_NODE_STAT
+                            )
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt

@@ -0,0 +1,59 @@
+   BenchmarkTestMultinode_Jenkinsfile.run()
+      BenchmarkTestMultinode_Jenkinsfile.pipeline(groovy.lang.Closure)
+         BenchmarkTestMultinode_Jenkinsfile.echo(Executing on agent [label:none])
+         BenchmarkTestMultinode_Jenkinsfile.stage(test-with-security, groovy.lang.Closure)
+            BenchmarkTestMultinode_Jenkinsfile.script(groovy.lang.Closure)
+               BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
+                  downloadBuildManifest.legacySCM(groovy.lang.Closure)
+                  downloadBuildManifest.library({identifier=jenkins@main, retriever=null})
+                  downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
+                  BuildManifest.asBoolean()
+               BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, userTag=key1:value1,security-enabled:true, workloadParams=key2:value2, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true})
+                  runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
+                  runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
+                  runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
+                  BuildManifest.asBoolean()
+                  runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runBenchmarkTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true})
+                        runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true})
+                  runBenchmarkTestScript.string({credentialsId=beta_datastore_username, variable=DATASTORE_USER})
+                  runBenchmarkTestScript.string({credentialsId=beta_datastore_password, variable=DATASTORE_PASSWORD})
+                  runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure)
+                     runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini})
+                     runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=})
+                  BuildManifest.getArtifactBuildId()
+                  BuildManifest.getArtifactArchitecture()
+                  BuildManifest.getCommitId(OpenSearch)
+                  runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true    --use-50-percent-heap --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3    --workload-params key2:value2 --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 )
+         BenchmarkTestMultinode_Jenkinsfile.stage(test-without-security, groovy.lang.Closure)
+            BenchmarkTestMultinode_Jenkinsfile.script(groovy.lang.Closure)
+               BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
+                  downloadBuildManifest.legacySCM(groovy.lang.Closure)
+                  downloadBuildManifest.library({identifier=jenkins@main, retriever=null})
+                  downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
+                  BuildManifest.asBoolean()
+               BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, userTag=key1:value1,security-enabled:false, workloadParams=key2:value2, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true})
+                  runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
+                  runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
+                  runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
+                  BuildManifest.asBoolean()
+                  runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
+                  runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
+                  runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
+                     runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        runBenchmarkTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true})
+                        runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true})
+                  runBenchmarkTestScript.string({credentialsId=beta_datastore_username, variable=DATASTORE_USER})
+                  runBenchmarkTestScript.string({credentialsId=beta_datastore_password, variable=DATASTORE_PASSWORD})
+                  runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure)
+                     runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini})
+                     runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=})
+                  BuildManifest.getArtifactBuildId()
+                  BuildManifest.getArtifactArchitecture()
+                  BuildManifest.getCommitId(OpenSearch)
+                  runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security   --use-50-percent-heap --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3    --workload-params key2:value2 --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 )