-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add pipeline for flaky test runner job (#46740)
* WIP Jenkinsfile for flaky test runner * Fix syntax * A few more jenkinsfile fixes * A few more fixes * Can't round numbers in Jenkins groovy sandbox apparently * More fixes * Only do build_kbn_tp_sample_panel_action once during flaky testing * Fix path and try setting a different JOB value * Keep track of flaky test failures * Introduce flaky test for testing * Some flaky test pipeline cleanup * Fix a couple of issues with flaky test failure tracking * Update flaky test runner build name/desc with metadata * Revert "Introduce flaky test for testing" This reverts commit 202e9d86cf130ef37aff1817781a5b694aa2aa32. * Try adding a local shared library * Move local library loading logic to kibana pipeline library * Move shared groovy code to shared library * Add missed file * Add ability to specify multiple agents for flaky test runner * Update kibana-pipeline-library version * Fix bug causing early exit for oss ciGroup1
- Loading branch information
1 parent
e5c9bfe
commit 5246b76
Showing
8 changed files
with
411 additions
and
281 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,113 @@ | ||
#!/bin/groovy | ||
|
||
library 'kibana-pipeline-library' | ||
kibanaLibrary.load() | ||
|
||
// Looks like 'oss:ciGroup:1' or 'oss:firefoxSmoke' | ||
def JOB_PARTS = params.CI_GROUP.split(':') | ||
def IS_XPACK = JOB_PARTS[0] == 'xpack' | ||
def JOB = JOB_PARTS[1] | ||
def CI_GROUP = JOB_PARTS[2] ?: '' | ||
|
||
def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP) | ||
|
||
def workerFailures = [] | ||
|
||
currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24) | ||
currentBuild.description = "${params.CI_GROUP}<br />Executions: ${params.NUMBER_EXECUTIONS}" | ||
|
||
// Note: If you increase agent count, it will execute NUMBER_EXECUTIONS per agent. It will not divide them up amongst the agents | ||
// e.g. NUMBER_EXECUTIONS = 25, agentCount = 4 results in 100 total executions | ||
def agentCount = 1 | ||
|
||
stage("Kibana Pipeline") { | ||
timeout(time: 180, unit: 'MINUTES') { | ||
timestamps { | ||
ansiColor('xterm') { | ||
def agents = [:] | ||
for(def agentNumber = 1; agentNumber <= agentCount; agentNumber++) { | ||
agents["agent-${agentNumber}"] = { | ||
catchError { | ||
kibanaPipeline.withWorkers('flaky-test-runner', { | ||
if (!IS_XPACK) { | ||
kibanaPipeline.buildOss() | ||
if (CI_GROUP == '1') { | ||
runbld "./test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh" | ||
} | ||
} else { | ||
kibanaPipeline.buildXpack() | ||
} | ||
}, getWorkerMap(agentNumber, params.NUMBER_EXECUTIONS.toInteger(), worker, workerFailures))() | ||
} | ||
} | ||
} | ||
|
||
parallel(agents) | ||
|
||
currentBuild.description += ", Failures: ${workerFailures.size()}" | ||
|
||
if (workerFailures.size() > 0) { | ||
print "There were ${workerFailures.size()} test suite failures." | ||
print "The executions that failed were:" | ||
print workerFailures.join("\n") | ||
print "Please check 'Test Result' and 'Pipeline Steps' pages for more info" | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
||
def getWorkerFromParams(isXpack, job, ciGroup) { | ||
if (!isXpack) { | ||
if (job == 'firefoxSmoke') { | ||
return kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld './test/scripts/jenkins_firefox_smoke.sh' }) | ||
} else if(job == 'visualRegression') { | ||
return kibanaPipeline.getPostBuildWorker('visualRegression', { runbld './test/scripts/jenkins_visual_regression.sh' }) | ||
} else { | ||
return kibanaPipeline.getOssCiGroupWorker(ciGroup) | ||
} | ||
} | ||
|
||
if (job == 'firefoxSmoke') { | ||
return kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld './test/scripts/jenkins_xpack_firefox_smoke.sh' }) | ||
} else if(job == 'visualRegression') { | ||
return kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld './test/scripts/jenkins_xpack_visual_regression.sh' }) | ||
} else { | ||
return kibanaPipeline.getXpackCiGroupWorker(ciGroup) | ||
} | ||
} | ||
|
||
def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 14) { | ||
def workerMap = [:] | ||
def numberOfWorkers = Math.min(numberOfExecutions, maxWorkers) | ||
|
||
for(def i = 1; i <= numberOfWorkers; i++) { | ||
def workerExecutions = numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0) | ||
|
||
workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber -> | ||
for(def j = 0; j < workerExecutions; j++) { | ||
print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}" | ||
withEnv(["JOB=agent-${agentNumber}-worker-${workerNumber}-${j}"]) { | ||
catchError { | ||
try { | ||
worker(workerNumber) | ||
} catch (ex) { | ||
workerFailures << "agent-${agentNumber} worker-${workerNumber}-${j}" | ||
throw ex | ||
} | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
||
return workerMap | ||
} | ||
|
||
def trunc(str, length) { | ||
if (str.size() >= length) { | ||
return str.take(length) + "..." | ||
} | ||
|
||
return str; | ||
} |
Oops, something went wrong.