From 9e738e4ec993e00abd0826f584ef064c3c636ace Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 30 Oct 2019 17:24:28 -0400 Subject: [PATCH] Flaky test runner improvements/fixes (#49367) * Allocate multiple flaky test runner agents as executions grow * WIP support for deleting kibana install dir during CI * Add setup script for testing scripts * Add REMOVE_KIBANA_INSTALL_DIR=1 to flaky test runner * Change flaky test runner worker processes from 8 to 12 --- .ci/Jenkinsfile_flaky | 38 +++++++++++++++++++------- test/scripts/jenkins_ci_group.sh | 10 +------ test/scripts/jenkins_test_setup.sh | 17 ++++++++++++ test/scripts/jenkins_xpack_ci_group.sh | 10 +------ 4 files changed, 47 insertions(+), 28 deletions(-) create mode 100644 test/scripts/jenkins_test_setup.sh diff --git a/.ci/Jenkinsfile_flaky b/.ci/Jenkinsfile_flaky index 3f77243da2c1b..e1cbac0528b1f 100644 --- a/.ci/Jenkinsfile_flaky +++ b/.ci/Jenkinsfile_flaky @@ -8,26 +8,28 @@ def JOB_PARTS = params.CI_GROUP.split(':') def IS_XPACK = JOB_PARTS[0] == 'xpack' def JOB = JOB_PARTS[1] def CI_GROUP = JOB_PARTS.size() > 2 ? JOB_PARTS[2] : '' +def EXECUTIONS = params.NUMBER_EXECUTIONS.toInteger() +def AGENT_COUNT = getAgentCount(EXECUTIONS) def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP) def workerFailures = [] currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24) -currentBuild.description = "${params.CI_GROUP}
Executions: ${params.NUMBER_EXECUTIONS}" - -// Note: If you increase agent count, it will execute NUMBER_EXECUTIONS per agent. It will not divide them up amongst the agents -// e.g. NUMBER_EXECUTIONS = 25, agentCount = 4 results in 100 total executions -def agentCount = 1 +currentBuild.description = "${params.CI_GROUP}
Agents: ${AGENT_COUNT}
Executions: ${params.NUMBER_EXECUTIONS}" stage("Kibana Pipeline") { timeout(time: 180, unit: 'MINUTES') { timestamps { ansiColor('xterm') { def agents = [:] - for(def agentNumber = 1; agentNumber <= agentCount; agentNumber++) { + for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) { + def agentNumberInside = agentNumber + def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0) agents["agent-${agentNumber}"] = { catchError { + print "Agent ${agentNumberInside} - ${agentExecutions} executions" + kibanaPipeline.withWorkers('flaky-test-runner', { if (!IS_XPACK) { kibanaPipeline.buildOss() @@ -37,7 +39,7 @@ stage("Kibana Pipeline") { } else { kibanaPipeline.buildXpack() } - }, getWorkerMap(agentNumber, params.NUMBER_EXECUTIONS.toInteger(), worker, workerFailures))() + }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))() } } } @@ -77,9 +79,9 @@ def getWorkerFromParams(isXpack, job, ciGroup) { } } -def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 14) { +def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkerProcesses = 12) { def workerMap = [:] - def numberOfWorkers = Math.min(numberOfExecutions, maxWorkers) + def numberOfWorkers = Math.min(numberOfExecutions, maxWorkerProcesses) for(def i = 1; i <= numberOfWorkers; i++) { def workerExecutions = numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0) @@ -87,7 +89,10 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber -> for(def j = 0; j < workerExecutions; j++) { print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}" - withEnv(["JOB=agent-${agentNumber}-worker-${workerNumber}-${j}"]) { + withEnv([ + "JOB=agent-${agentNumber}-worker-${workerNumber}-${j}", + "REMOVE_KIBANA_INSTALL_DIR=1", + ]) { catchError { try { worker(workerNumber) @@ -104,6 +109,11 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor return workerMap } +def getAgentCount(executions) { + // Increase agent count every 24 worker processess, up to 3 agents maximum + return Math.min(3, 1 + floor(executions/24)) +} + def trunc(str, length) { if (str.size() >= length) { return str.take(length) + "..." @@ -111,3 +121,11 @@ def trunc(str, length) { return str; } + +// All of the real rounding/truncating methods are sandboxed +def floor(num) { + return num + .toString() + .split('\\.')[0] + .toInteger() +} diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh index bd26ec1ff4cc1..2cd61ee4b7155 100755 --- a/test/scripts/jenkins_ci_group.sh +++ b/test/scripts/jenkins_ci_group.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash -set -e - -if [[ -z "$IS_PIPELINE_JOB" ]] ; then - trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT -else - source src/dev/ci_setup/setup_env.sh -fi - -export TEST_BROWSER_HEADLESS=1 +source test/scripts/jenkins_test_setup.sh if [[ -z "$IS_PIPELINE_JOB" ]] ; then yarn run grunt functionalTests:ensureAllTestsInCiGroup; diff --git a/test/scripts/jenkins_test_setup.sh b/test/scripts/jenkins_test_setup.sh new file mode 100644 index 0000000000000..cca5ed013fd88 --- /dev/null +++ b/test/scripts/jenkins_test_setup.sh @@ -0,0 +1,17 @@ +set -e + +function post_work() { + set +e + + if [[ -z "$REMOVE_KIBANA_INSTALL_DIR" && -z "$KIBANA_INSTALL_DIR" && -d "$KIBANA_INSTALL_DIR" ]]; then + rm -rf "$REMOVE_KIBANA_INSTALL_DIR" + fi +} + +trap 'post_work' EXIT + +export TEST_BROWSER_HEADLESS=1 + +if [[ -n "$IS_PIPELINE_JOB" ]] ; then + source src/dev/ci_setup/setup_env.sh +fi diff --git a/test/scripts/jenkins_xpack_ci_group.sh b/test/scripts/jenkins_xpack_ci_group.sh index 83c0042df4315..dfefe578f3e07 100755 --- a/test/scripts/jenkins_xpack_ci_group.sh +++ b/test/scripts/jenkins_xpack_ci_group.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash -set -e - -if [[ -z "$IS_PIPELINE_JOB" ]] ; then - trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT -else - source src/dev/ci_setup/setup_env.sh -fi - -export TEST_BROWSER_HEADLESS=1 +source test/scripts/jenkins_test_setup.sh if [[ -z "$IS_PIPELINE_JOB" ]] ; then echo " -> Ensuring all functional tests are in a ciGroup"