diff --git a/Jenkinsfile b/Jenkinsfile index 1ca7e81477d..230e7e3e71a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -250,6 +250,8 @@ def withBeatsEnv(Map args = [:], Closure body) { git config --global user.name "beatsmachine" fi''') } + // Skip to upload the generated files by default. + def upload = false try { // Add more stability when dependencies are not accessible temporarily // See https://github.com/elastic/beats/issues/21609 @@ -259,9 +261,13 @@ def withBeatsEnv(Map args = [:], Closure body) { cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true) } body() + } catch(err) { + // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage + upload = true + error("Error '${err.toString()}'") } finally { if (archive) { - archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id) + archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) } // Tear down the setup for the permamnent workers. catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { @@ -339,19 +345,38 @@ def archiveTestOutput(Map args = [:]) { script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;') } else { log(level: 'INFO', text: 'Delete folders that are causing exceptions (See JENKINS-58421) is disabled for Windows.') } junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, stashedTestReports: stashedTestReports, id: args.id) - tar(file: "test-build-artifacts-${args.id}.tgz", dir: '.', archive: true, allowMissing: true) + if (args.upload) { + tarAndUploadArtifacts(file: "test-build-artifacts-${args.id}.tgz", location: '.') + } } - catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() - log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") - if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() - tar(file: "${name}.tgz", archive: true, dir: folder) + if (args.upload) { + catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { + def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() + log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") + if (folder.trim()) { + // TODO: nodeOS() should support ARM + def os_suffix = isArm() ? 'linux' : nodeOS() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix + tarAndUploadArtifacts(file: "${name}.tgz", location: folder) + } } } } } +/** +* Wrapper to tar and upload artifacts to Google Storage to avoid killing the +* disk space of the jenkins instance +*/ +def tarAndUploadArtifacts(Map args = [:]) { + tar(file: args.file, dir: args.location, archive: false, allowMissing: true) + googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}", + credentialsId: "${JOB_GCS_CREDENTIALS}", + pattern: "${args.file}", + sharedPublicly: true, + showInline: true) +} + /** * Replace the slashes in the directory in case there are nested folders. */