From 6b367e57d6fe69b0e03f5ad989726660015e7685 Mon Sep 17 00:00:00 2001 From: Michael Oviedo Date: Thu, 8 Aug 2024 13:32:25 -0700 Subject: [PATCH 1/2] add compare logic to opensearch build libraries (#478) Signed-off-by: Michael Oviedo --- build.gradle | 2 +- tests/jenkins/TestRunCompareTestScript.groovy | 74 ++++++++++ .../jobs/CompareBenchmarkRun_Jenkinsfile.txt | 2 +- tests/jenkins/jobs/CompareTest_Jenkinsfile | 45 +++++++ .../jenkins/jobs/CompareTest_Jenkinsfile.txt | 39 ++++++ .../RunBenchmarkTestScriptLibTest.groovy | 8 +- .../RunCompareTestScriptLibTest.groovy | 82 +++++++++++ vars/getCompareBenchmarkIds.groovy | 2 +- vars/runBenchmarkTestScript.groovy | 127 +++++++++++------- 9 files changed, 326 insertions(+), 55 deletions(-) create mode 100644 tests/jenkins/TestRunCompareTestScript.groovy create mode 100644 tests/jenkins/jobs/CompareTest_Jenkinsfile create mode 100644 tests/jenkins/jobs/CompareTest_Jenkinsfile.txt create mode 100644 tests/jenkins/lib-testers/RunCompareTestScriptLibTest.groovy diff --git a/build.gradle b/build.gradle index f38838be..5f595b28 100644 --- a/build.gradle +++ b/build.gradle @@ -127,7 +127,7 @@ jacocoTestReport { } } -String version = '6.8.0' +String version = '6.8.1' task updateVersion { doLast { diff --git a/tests/jenkins/TestRunCompareTestScript.groovy b/tests/jenkins/TestRunCompareTestScript.groovy new file mode 100644 index 00000000..1eede6d5 --- /dev/null +++ b/tests/jenkins/TestRunCompareTestScript.groovy @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat + +class TestRunCompareTestScript extends BuildPipelineTest { + @Before + void setUp() { + this.registerLibTester(new RunCompareTestScriptLibTester( + 'compare', + '12345', + '54321', + 'csv', + 'left', + '/results/final_results', + 'true', + 'compare-tester', + 'tests/data/opensearch-1.3.0-bundle.yml' + )) + super.setUp() + } + + @Test + public void testRunCompareTestScript_PipelineSingleNode() { + super.testPipeline("tests/jenkins/jobs/CompareTest_Jenkinsfile") + } + + @Test + void testRunCompareTestScript_verifyScriptExecutionSingleNode() { + runScript("tests/jenkins/jobs/CompareTest_Jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(2)) + assertThat(testScriptCommands, hasItem( + "set +x && ./test.sh benchmark-test compare 12345 54321 --benchmark-config /tmp/workspace/benchmark.ini --suffix compare-tester" + )) + assertThat(testScriptCommands, hasItem( + "set +x && ./test.sh benchmark-test compare 12345 54321 --benchmark-config /tmp/workspace/benchmark.ini --suffix compare-tester --results-format=csv --results-numbers-align=left --results-file=/results/final_results --show-in-results=true" + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } + +} \ No newline at end of file diff --git a/tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt b/tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt index 14b653cb..6507bd34 100644 --- a/tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt +++ b/tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt @@ -6,7 +6,7 @@ CompareBenchmarkRun_Jenkinsfile.getCompareBenchmarkIds({baselineClusterConfig=test-baseline-config, distributionVersion=3.0.0, workload=big5, pullRequestNumber=12345}) getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) - getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-endpoint, variable=DATASTORE_ENDPOINT}) + getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-nlb-endpoint, variable=DATASTORE_ENDPOINT}) getCompareBenchmarkIds.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD, DATASTORE_ENDPOINT], groovy.lang.Closure) getCompareBenchmarkIds.sh({script= curl -X POST "https://DATASTORE_ENDPOINT/benchmark-results-*/_search" -ku DATASTORE_USER:DATASTORE_PASSWORD -H 'Content-Type: application/json' -d '{ diff --git a/tests/jenkins/jobs/CompareTest_Jenkinsfile b/tests/jenkins/jobs/CompareTest_Jenkinsfile new file mode 100644 index 00000000..b731e2cb --- /dev/null +++ b/tests/jenkins/jobs/CompareTest_Jenkinsfile @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('compare-test') { + parallel { + stage('test-with-all-args') { + steps { + script { + runBenchmarkTestScript(command: COMMAND, + baseline: BASELINE, + contender: CONTENDER, + results_format: RESULTS_FORMAT, + results_numbers_align: RESULTS_NUMBERS_ALIGN, + results_file: RESULTS_FILE, + show_in_results: SHOW_IN_RESULTS, + suffix: SUFFIX, + bundleManifest: BUNDLE_MANIFEST + ) + } + } + } + stage('test-with-base-args') { + steps { + script { + runBenchmarkTestScript(command: COMMAND, + baseline: BASELINE, + contender: CONTENDER, + suffix: SUFFIX, + bundleManifest: BUNDLE_MANIFEST + ) + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/jenkins/jobs/CompareTest_Jenkinsfile.txt b/tests/jenkins/jobs/CompareTest_Jenkinsfile.txt new file mode 100644 index 00000000..2d3d0a5c --- /dev/null +++ b/tests/jenkins/jobs/CompareTest_Jenkinsfile.txt @@ -0,0 +1,39 @@ + CompareTest_Jenkinsfile.run() + CompareTest_Jenkinsfile.pipeline(groovy.lang.Closure) + CompareTest_Jenkinsfile.echo(Executing on agent [label:none]) + CompareTest_Jenkinsfile.stage(test-with-all-args, groovy.lang.Closure) + CompareTest_Jenkinsfile.script(groovy.lang.Closure) + CompareTest_Jenkinsfile.runBenchmarkTestScript({command=compare, baseline=12345, contender=54321, results_format=csv, results_numbers_align=left, results_file=/results/final_results, show_in_results=true, suffix=compare-tester, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) + runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test compare 12345 54321 --benchmark-config /tmp/workspace/benchmark.ini --suffix compare-tester --results-format=csv --results-numbers-align=left --results-file=/results/final_results --show-in-results=true) + CompareTest_Jenkinsfile.stage(test-with-base-args, groovy.lang.Closure) + CompareTest_Jenkinsfile.script(groovy.lang.Closure) + CompareTest_Jenkinsfile.runBenchmarkTestScript({command=compare, baseline=12345, contender=54321, suffix=compare-tester, bundleManifest=tests/data/opensearch-1.3.0-bundle.yml}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) + runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(set +x && ./test.sh benchmark-test compare 12345 54321 --benchmark-config /tmp/workspace/benchmark.ini --suffix compare-tester) diff --git a/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy index 4660a6cc..8a1e22ec 100644 --- a/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy @@ -41,10 +41,10 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ private String captureSegmentReplicationStat private String telemetryParams - public RunBenchmarkTestScriptLibTester(command, bundleManifest, distributionUrl, distributionVersion, insecure, workload, singleNode, minDistribution, use50PercentHeap, - enableRemoteStore, managerNodeCount, dataNodeCount, dataInstanceType, userTag, workloadParams, - testProcedure, excludeTasks, includeTasks, - additionalConfig, captureNodeStat, captureSegmentReplicationStat, telemetryParams){ + public RunBenchmarkTestScriptLibTester(command, bundleManifest, distributionUrl, distributionVersion, insecure, workload, + singleNode, minDistribution, use50PercentHeap, enableRemoteStore, managerNodeCount, + dataNodeCount, dataInstanceType, userTag, workloadParams, testProcedure, excludeTasks, + includeTasks, additionalConfig, captureNodeStat, captureSegmentReplicationStat, telemetryParams){ this.command = command this.bundleManifest = bundleManifest this.distributionUrl = distributionUrl diff --git a/tests/jenkins/lib-testers/RunCompareTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunCompareTestScriptLibTest.groovy new file mode 100644 index 00000000..073840a0 --- /dev/null +++ b/tests/jenkins/lib-testers/RunCompareTestScriptLibTest.groovy @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat + +class RunCompareTestScriptLibTester extends LibFunctionTester{ + + private String command + private String baseline + private String contender + private String results_format + private String results_numbers_align + private String results_file + private String show_in_results + private String suffix + private String bundleManifest + + public RunCompareTestScriptLibTester(command, baseline, contender, results_format, + results_numbers_align, results_file, show_in_results, + suffix, bundleManifest){ + this.command = command + this.baseline = baseline + this.contender = contender + this.results_format = results_format + this.results_numbers_align = results_numbers_align + this.results_file = results_file + this.show_in_results = show_in_results + this.suffix = suffix + this.bundleManifest = bundleManifest + } + + + @Override + String libFunctionName() { + return 'runBenchmarkTestScript' + } + + @Override + void parameterInvariantsAssertions(Object call) {} + + @Override + boolean expectedParametersMatcher(Object call) { + if (!this.bundleManifest.isEmpty()) { + return call.args.bundleManifest.first().toString().equals(this.bundleManifest) + } + return call.args.distributionUrl.first().toString().equals(this.distributionUrl) + } + + @Override + void configure(Object helper, Object binding) { + binding.setVariable('COMMAND', command) + binding.setVariable('SUFFIX', suffix) + binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('BASELINE', baseline) + binding.setVariable('CONTENDER', contender) + binding.setVariable('RESULTS_FORMAT', results_format) + binding.setVariable('RESULTS_NUMBERS_ALIGN', results_numbers_align) + binding.setVariable('RESULTS_FILE', results_file) + binding.setVariable('SHOW_IN_RESULTS', show_in_results) + binding.setVariable('BUNDLE_MANIFEST', bundleManifest) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod("s3Download", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod("withCredentials", [Map]) + } +} diff --git a/vars/getCompareBenchmarkIds.groovy b/vars/getCompareBenchmarkIds.groovy index 31892e37..0283ce50 100644 --- a/vars/getCompareBenchmarkIds.groovy +++ b/vars/getCompareBenchmarkIds.groovy @@ -31,7 +31,7 @@ Map call(Map args = [:]) { String getBaselineTestExecutionId(baselineClusterConfig, distributionVersion, workload) { withCredentials([string(credentialsId: 'benchmark-metrics-datastore-user', variable: 'DATASTORE_USER'), string(credentialsId: 'benchmark-metrics-datastore-password', variable: 'DATASTORE_PASSWORD'), - string(credentialsId: 'benchmark-metrics-datastore-endpoint', variable: 'DATASTORE_ENDPOINT')]) { + string(credentialsId: 'benchmark-metrics-datastore-nlb-endpoint', variable: 'DATASTORE_ENDPOINT')]) { def curlCommand = """ curl -X POST "https://${DATASTORE_ENDPOINT}/benchmark-results-*/_search" -ku ${DATASTORE_USER}:${DATASTORE_PASSWORD} -H 'Content-Type: application/json' -d '{ "size": 1, diff --git a/vars/runBenchmarkTestScript.groovy b/vars/runBenchmarkTestScript.groovy index c1afffa3..7b3b6d3a 100644 --- a/vars/runBenchmarkTestScript.groovy +++ b/vars/runBenchmarkTestScript.groovy @@ -9,6 +9,7 @@ /** Library to execute benchmark-test using opensearch-benchmark and opensearch-cluster-cdk * * @param Map args = [:] args A map of the following parameters + * @param args.command - Name of command to run. 'execute-test' or 'compare'. * @param args.bundleManifest - OpenSearch bundle manifest url. * @param args.distributionUrl - Download link for the OpenSearch bundle tarball. * @param args.distributionVersion - Provide OpenSearch version if using distributionUrl param @@ -38,8 +39,15 @@ * @param args.userTag - Additional metadata tags to be added to benchmark run metrics, e.g., run-type:adhoc,arch:x64 * @param args.configName - Name of the config file that needs to be downloaded from S3 bucket, default is config.yml. * @param args.telemetryParams - Allows to set parameters for telemetry devices such as node-stat etc., e.g. {"node-stats-include-indices": "true"} + * @param args.baseline - The baseline TestExecution ID used to compare the contender TestExecution. + * @param args.contender - The TestExecution ID for the contender being compared to the baseline. + * @param args.results_format - Defines the output format for the command line results, either markdown or csv. Default is markdown. + * @param args.results_numbers_align - Defines the column number alignment for when the compare command outputs results. Default is right. + * @param args.results_file - When provided a file path, writes the compare results to the file indicated in the path. + * @param args.show_in_results - Determines whether or not to include the comparison in the results file. */ void call(Map args = [:]) { + lib = library(identifier: 'jenkins@main', retriever: legacySCM(scm)) def buildManifest = null @@ -50,9 +58,9 @@ void call(Map args = [:]) { config_name = isNullOrEmpty(args.config) ? 'config.yml' : args.config benchmark_config = 'benchmark.ini' withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC'), - string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { + string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - if(isNullOrEmpty(args.endpoint)) { + if(isNullOrEmpty(args.endpoint) && args.command == 'execute-test') { s3Download(file: 'config.yml', bucket: "${ARTIFACT_BUCKET_NAME}", path: "${BENCHMARK_TEST_CONFIG_LOCATION}/${config_name}", force: true) } s3Download(file: 'benchmark.ini', bucket: "${ARTIFACT_BUCKET_NAME}", path: "${BENCHMARK_TEST_CONFIG_LOCATION}/${benchmark_config}", force: true) @@ -60,55 +68,78 @@ void call(Map args = [:]) { /*Added sleep to let the file get downloaded first before write happens. Without the sleep the write is happening in parallel to download resulting in file not found error. To avoid pip install conflict errors when runnin with and without security run in parallel add enough gap between execution. - */ - if (args.insecure.toBoolean()) { - sleep(5) - } else { - sleep(120) - } + */ } } + editBenchmarkConfig("${WORKSPACE}/benchmark.ini") - String userTags = getMetadataTags(args.userTag.toString(), buildManifest) - - def command = [ - './test.sh', - 'benchmark-test', - args.command, - isNullOrEmpty(args.bundleManifest) ? "" : "--bundle-manifest ${args.bundleManifest}", - isNullOrEmpty(args.distributionUrl) ? "" : "--distribution-url ${args.distributionUrl}", - isNullOrEmpty(args.distributionVersion) ? "" : "--distribution-version ${args.distributionVersion}", - isNullOrEmpty(args.endpoint) ? "" : "--cluster-endpoint ${args.endpoint}", - isNullOrEmpty(args.endpoint) ? "--config ${WORKSPACE}/config.yml" : "", - "--workload ${args.workload}", - "--benchmark-config ${WORKSPACE}/benchmark.ini", - "--user-tag ${userTags}", - args.insecure?.toBoolean() ? "--without-security" : "", - isNullOrEmpty(args.username) ? "" : "--username ${args.username}", - isNullOrEmpty(args.password) ? "" : "--password ${args.password}", - args.singleNode?.toBoolean() ? "--single-node" : "", - args.minDistribution?.toBoolean() ? "--min-distribution" : "", - args.use50PercentHeap?.toBoolean() ? "--use-50-percent-heap" : "", - args.enableRemoteStore?.toBoolean() ? "--enable-remote-store" : "", - args.captureNodeStat?.toBoolean() ? "--capture-node-stat" : "", - args.captureSegmentReplicationStat?.toBoolean() ? "--capture-segment-replication-stat" : "", - isNullOrEmpty(args.suffix) ? "" : "--suffix ${args.suffix}", - isNullOrEmpty(args.managerNodeCount) ? "" : "--manager-node-count ${args.managerNodeCount}", - isNullOrEmpty(args.dataNodeCount) ? "" : "--data-node-count ${args.dataNodeCount}", - isNullOrEmpty(args.clientNodeCount) ? "" : "--client-node-count ${args.clientNodeCount}", - isNullOrEmpty(args.ingestNodeCount) ? "" : "--ingest-node-count ${args.ingestNodeCount}", - isNullOrEmpty(args.mlNodeCount) ? "" : "--ml-node-count ${args.mlNodeCount}", - isNullOrEmpty(args.dataInstanceType) ? "" : "--data-instance-type ${args.dataInstanceType}", - isNullOrEmpty(args.workloadParams) ? "" : "--workload-params '${args.workloadParams}'", - isNullOrEmpty(args.testProcedure) ? "" : "--test-procedure ${args.testProcedure}", - isNullOrEmpty(args.excludeTasks) ? "" : "--exclude-tasks ${args.excludeTasks}", - isNullOrEmpty(args.includeTasks) ? "" : "--include-tasks ${args.includeTasks}", - isNullOrEmpty(args.additionalConfig) ? "" : "--additional-config ${args.additionalConfig}", - isNullOrEmpty(args.dataStorageSize) ? "" : "--data-node-storage ${args.dataStorageSize}", - isNullOrEmpty(args.mlStorageSize) ? "" : "--ml-node-storage ${args.mlStorageSize}", - isNullOrEmpty(args.jvmSysProps) ? "" : "--jvm-sys-props ${args.jvmSysProps}", - isNullOrEmpty(args.telemetryParams) ? "" : "--telemetry-params '${args.telemetryParams}'" - ].join(' ').trim() + + String command = '' + + if(args.command == 'execute-test') { + + if (args.insecure.toBoolean()) { + sleep(5) + } else { + sleep(120) + } + + String userTags = getMetadataTags(args.userTag.toString(), buildManifest) + + command = [ + './test.sh', + 'benchmark-test', + args.command, + isNullOrEmpty(args.bundleManifest) ? "" : "--bundle-manifest ${args.bundleManifest}", + isNullOrEmpty(args.distributionUrl) ? "" : "--distribution-url ${args.distributionUrl}", + isNullOrEmpty(args.distributionVersion) ? "" : "--distribution-version ${args.distributionVersion}", + isNullOrEmpty(args.endpoint) ? "" : "--cluster-endpoint ${args.endpoint}", + isNullOrEmpty(args.endpoint) ? "--config ${WORKSPACE}/config.yml" : "", + "--workload ${args.workload}", + "--benchmark-config ${WORKSPACE}/benchmark.ini", + "--user-tag ${userTags}", + args.insecure?.toBoolean() ? "--without-security" : "", + isNullOrEmpty(args.username) ? "" : "--username ${args.username}", + isNullOrEmpty(args.password) ? "" : "--password ${args.password}", + args.singleNode?.toBoolean() ? "--single-node" : "", + args.minDistribution?.toBoolean() ? "--min-distribution" : "", + args.use50PercentHeap?.toBoolean() ? "--use-50-percent-heap" : "", + args.enableRemoteStore?.toBoolean() ? "--enable-remote-store" : "", + args.captureNodeStat?.toBoolean() ? "--capture-node-stat" : "", + args.captureSegmentReplicationStat?.toBoolean() ? "--capture-segment-replication-stat" : "", + isNullOrEmpty(args.suffix) ? "" : "--suffix ${args.suffix}", + isNullOrEmpty(args.managerNodeCount) ? "" : "--manager-node-count ${args.managerNodeCount}", + isNullOrEmpty(args.dataNodeCount) ? "" : "--data-node-count ${args.dataNodeCount}", + isNullOrEmpty(args.clientNodeCount) ? "" : "--client-node-count ${args.clientNodeCount}", + isNullOrEmpty(args.ingestNodeCount) ? "" : "--ingest-node-count ${args.ingestNodeCount}", + isNullOrEmpty(args.mlNodeCount) ? "" : "--ml-node-count ${args.mlNodeCount}", + isNullOrEmpty(args.dataInstanceType) ? "" : "--data-instance-type ${args.dataInstanceType}", + isNullOrEmpty(args.workloadParams) ? "" : "--workload-params '${args.workloadParams}'", + isNullOrEmpty(args.testProcedure) ? "" : "--test-procedure ${args.testProcedure}", + isNullOrEmpty(args.excludeTasks) ? "" : "--exclude-tasks ${args.excludeTasks}", + isNullOrEmpty(args.includeTasks) ? "" : "--include-tasks ${args.includeTasks}", + isNullOrEmpty(args.additionalConfig) ? "" : "--additional-config ${args.additionalConfig}", + isNullOrEmpty(args.dataStorageSize) ? "" : "--data-node-storage ${args.dataStorageSize}", + isNullOrEmpty(args.mlStorageSize) ? "" : "--ml-node-storage ${args.mlStorageSize}", + isNullOrEmpty(args.jvmSysProps) ? "" : "--jvm-sys-props ${args.jvmSysProps}", + isNullOrEmpty(args.telemetryParams) ? "" : "--telemetry-params '${args.telemetryParams}'" + ].join(' ').trim() + + } else if(args.command == 'compare') { + command = [ + './test.sh', + 'benchmark-test', + args.command, + args.baseline, + args.contender, + "--benchmark-config ${WORKSPACE}/benchmark.ini", + isNullOrEmpty(args.suffix) ? "" : "--suffix ${args.suffix}", + isNullOrEmpty(args.results_format) ? "" : "--results-format=${args.results_format}", + isNullOrEmpty(args.results_numbers_align) ? "" : "--results-numbers-align=${args.results_numbers_align}", + isNullOrEmpty(args.results_file) ? "" : "--results-file=${args.results_file}", + isNullOrEmpty(args.show_in_results) ? "" : "--show-in-results=${args.show_in_results}" + ].join(' ').trim() + } sh """set +x && ${command}""" From f0eeaff63f078d226065e75082d7108850a64c79 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Fri, 9 Aug 2024 09:36:20 -0700 Subject: [PATCH 2/2] Fix datastore endpoint secret name (#481) Signed-off-by: Rishabh Singh --- build.gradle | 2 +- tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt | 2 +- vars/getCompareBenchmarkIds.groovy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 5f595b28..4ba5981b 100644 --- a/build.gradle +++ b/build.gradle @@ -127,7 +127,7 @@ jacocoTestReport { } } -String version = '6.8.1' +String version = '6.8.2' task updateVersion { doLast { diff --git a/tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt b/tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt index 6507bd34..48706ac8 100644 --- a/tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt +++ b/tests/jenkins/jobs/CompareBenchmarkRun_Jenkinsfile.txt @@ -53,7 +53,7 @@ getCompareBenchmarkIds.echo(Latest test-execution-id: test-id) getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) - getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-endpoint, variable=DATASTORE_ENDPOINT}) + getCompareBenchmarkIds.string({credentialsId=benchmark-metrics-datastore-nlb-endpoint, variable=DATASTORE_ENDPOINT}) getCompareBenchmarkIds.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD, DATASTORE_ENDPOINT], groovy.lang.Closure) getCompareBenchmarkIds.sh({script= curl -X POST "https://DATASTORE_ENDPOINT/benchmark-results-*/_search" -ku DATASTORE_USER:DATASTORE_PASSWORD -H 'Content-Type: application/json' -d '{ diff --git a/vars/getCompareBenchmarkIds.groovy b/vars/getCompareBenchmarkIds.groovy index 0283ce50..9dae5745 100644 --- a/vars/getCompareBenchmarkIds.groovy +++ b/vars/getCompareBenchmarkIds.groovy @@ -84,7 +84,7 @@ String getBaselineTestExecutionId(baselineClusterConfig, distributionVersion, wo String getContenderTestExecutionId(pullRequestNumber) { withCredentials([string(credentialsId: 'benchmark-metrics-datastore-user', variable: 'DATASTORE_USER'), string(credentialsId: 'benchmark-metrics-datastore-password', variable: 'DATASTORE_PASSWORD'), - string(credentialsId: 'benchmark-metrics-datastore-endpoint', variable: 'DATASTORE_ENDPOINT')]) { + string(credentialsId: 'benchmark-metrics-datastore-nlb-endpoint', variable: 'DATASTORE_ENDPOINT')]) { def curlCommand = """ curl -X POST "https://${DATASTORE_ENDPOINT}/benchmark-results-*/_search" -ku ${DATASTORE_USER}:${DATASTORE_PASSWORD} -H 'Content-Type: application/json' -d '{ "size": 1,