Skip to content

Commit

Permalink
add compare logic to opensearch build libraries
Browse files Browse the repository at this point in the history 
Signed-off-by: Michael Oviedo <[email protected]>
  • Loading branch information
@OVI3D0
OVI3D0 committed Aug 7, 2024
1 parent f482df7 commit 445c1a3
Show file tree
Hide file tree
Showing 14 changed files with 312 additions and 53 deletions.
6 changes: 6 additions & 0 deletions tests/jenkins/TestRunBenchmarkTestScript.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest {
'cluster.indices.replication.strategy:SEGMENT',
'false',
'true',
'',
'',
'',
'',
'',
'',
''
))
super.setUp()
Expand Down
8 changes: 7 additions & 1 deletion tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,13 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest {
'cluster.indices.replication.strategy:SEGMENT',
'true',
'false',
'{"telemetry_setting":"value"}'
'{"telemetry_setting":"value"}',
'',
'',
'',
'',
'',
''
))
super.setUp()
}
Expand Down
6 changes: 6 additions & 0 deletions tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ class TestRunBenchmarkTestScriptNoManifest extends BuildPipelineTest {
'cluster.indices.replication.strategy:SEGMENT',
'false',
'true',
'',
'',
'',
'',
'',
'',
''
))
super.setUp()
Expand Down
93 changes: 93 additions & 0 deletions tests/jenkins/TestRunCompareTestScript.groovy
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

import jenkins.tests.BuildPipelineTest
import org.junit.Before
import org.junit.Test

import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
import static org.hamcrest.CoreMatchers.equalTo
import static org.hamcrest.CoreMatchers.hasItem
import static org.hamcrest.MatcherAssert.assertThat

class TestRunCompareTestScript extends BuildPipelineTest {
@Before
void setUp() {
this.registerLibTester(new RunBenchmarkTestScriptLibTester(
'compare',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'',
'12345',
'54321',
'csv',
'left',
'/results/final_results',
'true'
))
super.setUp()
}

@Test
public void testRunCompareTestScript_PipelineSingleNode() {
super.testPipeline("tests/jenkins/jobs/CompareTest_Jenkinsfile")
}

@Test
void testRunCompareTestScript_verifyScriptExecutionSingleNode() {
runScript("tests/jenkins/jobs/CompareTest_Jenkinsfile")

def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
shCommand -> shCommand.contains('./test.sh')
}

assertThat(testScriptCommands.size(), equalTo(2))
assertThat(testScriptCommands, hasItem(
"set +x && ./test.sh benchmark-test compare 12345 54321"
))
assertThat(testScriptCommands, hasItem(
"set +x && ./test.sh benchmark-test compare 12345 54321 --results-format=csv --results-numbers-align=left --results-file=/results/final_results --show-in-results=true"
))
}

def getCommandExecutions(methodName, command) {
def shCommands = helper.callStack.findAll {
call ->
call.methodName == methodName
}.
collect {
call ->
callArgsToString(call)
}.findAll {
shCommand ->
shCommand.contains(command)
}

return shCommands
}

}
15 changes: 13 additions & 2 deletions tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,13 @@ pipeline {
jvmSysProps: JVM_SYS_PROPS,
captureNodeStat: CAPTURE_NODE_STAT,
captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT,
telemetryParams: TELEMETRY_PARAMS
telemetryParams: TELEMETRY_PARAMS,
baseline: BASELINE,
contender: CONTENDER,
results_format: RESULTS_FORMAT,
results_numbers_align: RESULTS_NUMBERS_ALIGN,
results_file: RESULTS_FILE,
show_in_results: SHOW_IN_RESULTS
)
}
}
Expand Down Expand Up @@ -86,7 +92,12 @@ pipeline {
jvmSysProps: JVM_SYS_PROPS,
captureNodeStat: CAPTURE_NODE_STAT,
captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT,
telemetryParams: TELEMETRY_PARAMS
telemetryParams: TELEMETRY_PARAMS,
baseline: BASELINE,
contender: CONTENDER,
results_format: RESULTS_FORMAT,
results_numbers_align: RESULTS_NUMBERS_ALIGN,
show_in_results: SHOW_IN_RESULTS
)
}
}
Expand Down
4 changes: 2 additions & 2 deletions tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
BuildManifest.asBoolean()
BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}, baseline=, contender=, results_format=, results_numbers_align=, results_file=, show_in_results=})
runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
Expand Down Expand Up @@ -41,7 +41,7 @@
downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
BuildManifest.asBoolean()
BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}, baseline=, contender=, results_format=, results_numbers_align=, show_in_results=})
runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
Expand Down
15 changes: 13 additions & 2 deletions tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,13 @@ pipeline {
jvmSysProps: JVM_SYS_PROPS,
captureNodeStat: CAPTURE_NODE_STAT,
captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT,
telemetryParams: TELEMETRY_PARAMS
telemetryParams: TELEMETRY_PARAMS,
baseline: BASELINE,
contender: CONTENDER,
results_format: RESULTS_FORMAT,
results_numbers_align: RESULTS_NUMBERS_ALIGN,
results_file: RESULTS_FILE,
show_in_results: SHOW_IN_RESULTS
)
}
}
Expand Down Expand Up @@ -78,7 +84,12 @@ pipeline {
jvmSysProps: JVM_SYS_PROPS,
captureNodeStat: CAPTURE_NODE_STAT,
captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT,
telemetryParams: TELEMETRY_PARAMS
telemetryParams: TELEMETRY_PARAMS,
baseline: BASELINE,
contender: CONTENDER,
results_format: RESULTS_FORMAT,
results_numbers_align: RESULTS_NUMBERS_ALIGN,
show_in_results: SHOW_IN_RESULTS
)
}
}
Expand Down
4 changes: 2 additions & 2 deletions tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
BenchmarkTestNoManifest_Jenkinsfile.echo(Executing on agent [label:none])
BenchmarkTestNoManifest_Jenkinsfile.stage(test-with-security, groovy.lang.Closure)
BenchmarkTestNoManifest_Jenkinsfile.script(groovy.lang.Closure)
BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=, distributionUrl=https://www.example.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams=, testProcedure=custom-test-procedure, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=})
BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=, distributionUrl=https://www.example.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams=, testProcedure=custom-test-procedure, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=, baseline=, contender=, results_format=, results_numbers_align=, results_file=, show_in_results=})
runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
Expand All @@ -20,7 +20,7 @@
runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test execute-test --distribution-url https://www.example.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200)
BenchmarkTestNoManifest_Jenkinsfile.stage(test-without-security, groovy.lang.Closure)
BenchmarkTestNoManifest_Jenkinsfile.script(groovy.lang.Closure)
BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=, distributionUrl=https://www.example.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams=, testProcedure=custom-test-procedure, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=})
BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=, distributionUrl=https://www.example.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams=, testProcedure=custom-test-procedure, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=, baseline=, contender=, results_format=, results_numbers_align=, show_in_results=})
runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
Expand Down
8 changes: 7 additions & 1 deletion tests/jenkins/jobs/BenchmarkTest_Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,13 @@ pipeline {
jvmSysProps: JVM_SYS_PROPS,
captureNodeStat: CAPTURE_NODE_STAT,
captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT,
telemetryParams: TELEMETRY_PARAMS
telemetryParams: TELEMETRY_PARAMS,
baseline: BASELINE,
contender: CONTENDER,
results_format: RESULTS_FORMAT,
results_numbers_align: RESULTS_NUMBERS_ALIGN,
results_file: RESULTS_FILE,
show_in_results: SHOW_IN_RESULTS
)
}
}
Expand Down
2 changes: 1 addition & 1 deletion tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
BuildManifest.asBoolean()
BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=})
BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({command=execute-test, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=, baseline=, contender=, results_format=, results_numbers_align=, results_file=, show_in_results=})
runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
Expand Down
Loading

0 comments on commit 445c1a3

Please sign in to comment.