[Bug]: Library downloadBuildManifest error `curl: (23) Failed writi…

…ng body (0 != 4804)` (opensearch-project#468) Signed-off-by: Prudhvi Godithi <[email protected]>
peterzhuamazon · Jul 30, 2024 · 7ddabf2 · 7ddabf2
1 parent 3822ade
commit 7ddabf2
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 10 deletions.
diff --git a/tests/jenkins/TestRunBenchmarkTestScript.groovy b/tests/jenkins/TestRunBenchmarkTestScript.groovy
@@ -61,7 +61,7 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest {
 
         assertThat(curlCommands.size(), equalTo(2))
         assertThat(curlCommands, hasItem(
-                "curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString()
+                "curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString()
         ))
 
         def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll {

diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy
@@ -47,7 +47,7 @@ class TestRunPerfTestScript extends BuildPipelineTest {
 
         assertThat(curlCommands.size(), equalTo(2))
         assertThat(curlCommands, hasItem(
-            "curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString()
+            "curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml".toString()
         ))
 
         def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll {

diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt
@@ -6,7 +6,9 @@
                BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
                   downloadBuildManifest.legacySCM(groovy.lang.Closure)
                   downloadBuildManifest.library({identifier=jenkins@main, retriever=null})
-                  downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml"))
+                  downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
                   downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
                   BuildManifest.asBoolean()
                BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
@@ -34,7 +36,9 @@
                BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
                   downloadBuildManifest.legacySCM(groovy.lang.Closure)
                   downloadBuildManifest.library({identifier=jenkins@main, retriever=null})
-                  downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml"))
+                  downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
                   downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
                   BuildManifest.asBoolean()
                BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})

diff --git a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt
@@ -6,7 +6,9 @@
                BenchmarkTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
                   downloadBuildManifest.legacySCM(groovy.lang.Closure)
                   downloadBuildManifest.library({identifier=jenkins@main, retriever=null})
-                  downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml"))
+                  downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
                   downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
                   BuildManifest.asBoolean()
                BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=})
@@ -34,7 +36,9 @@
                BenchmarkTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
                   downloadBuildManifest.legacySCM(groovy.lang.Closure)
                   downloadBuildManifest.library({identifier=jenkins@main, retriever=null})
-                  downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml"))
+                  downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
                   downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
                   BuildManifest.asBoolean()
                BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=})

diff --git a/tests/jenkins/jobs/PerfTest_Jenkinsfile.txt b/tests/jenkins/jobs/PerfTest_Jenkinsfile.txt
@@ -6,7 +6,9 @@
                PerfTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
                   downloadBuildManifest.legacySCM(groovy.lang.Closure)
                   downloadBuildManifest.library({identifier=jenkins@main, retriever=null})
-                  downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml"))
+                  downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
                   downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
                   BuildManifest.asBoolean()
                PerfTest_Jenkinsfile.runPerfTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1, stackNameSuffix=307})
@@ -32,7 +34,9 @@
                PerfTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
                   downloadBuildManifest.legacySCM(groovy.lang.Closure)
                   downloadBuildManifest.library({identifier=jenkins@main, retriever=null})
-                  downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(mkdir -p $(dirname "tests/data/opensearch-1.3.0-bundle.yml"))
+                  downloadBuildManifest.sh(touch tests/data/opensearch-1.3.0-bundle.yml)
+                  downloadBuildManifest.sh(curl -sSL --retry 5 test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
                   downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
                   BuildManifest.asBoolean()
                PerfTest_Jenkinsfile.runPerfTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1, stackNameSuffix=307})

diff --git a/vars/downloadBuildManifest.groovy b/vars/downloadBuildManifest.groovy
@@ -8,8 +8,9 @@
  */
 def call(Map args = [:]) {
     def lib = library(identifier: "jenkins@main", retriever: legacySCM(scm))
-
-    sh "curl -sSL ${args.url} --output ${args.path}"
+    sh "mkdir -p \$(dirname \"${args.path}\")"
+    sh "touch ${args.path}"
+    sh "curl -sSL --retry 5 ${args.url} --output ${args.path}"
     def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: args.path))
     return buildManifestObj
 }