Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flaky test runner improvements/fixes #49367

Merged
merged 7 commits into from
Oct 30, 2019
Merged

Flaky test runner improvements/fixes #49367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e8ba96d
Allocate multiple flaky test runner agents as executions grow
brianseeders Oct 21, 2019
435aed5
WIP support for deleting kibana install dir during CI
brianseeders Oct 22, 2019
345d81b
Add setup script for testing scripts
brianseeders Oct 23, 2019
9823e63
Add REMOVE_KIBANA_INSTALL_DIR=1 to flaky test runner
brianseeders Oct 23, 2019
52908c6
Merge branch 'master' into flaky-test-runner-improvements
elasticmachine Oct 29, 2019
5d12ed9
Change flaky test runner worker processes from 8 to 12
brianseeders Oct 29, 2019
5e04ab8
Merge branch 'master' into flaky-test-runner-improvements
elasticmachine Oct 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 28 additions & 10 deletions .ci/Jenkinsfile_flaky
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,28 @@ def JOB_PARTS = params.CI_GROUP.split(':')
def IS_XPACK = JOB_PARTS[0] == 'xpack'
def JOB = JOB_PARTS[1]
def CI_GROUP = JOB_PARTS.size() > 2 ? JOB_PARTS[2] : ''
def EXECUTIONS = params.NUMBER_EXECUTIONS.toInteger()
def AGENT_COUNT = getAgentCount(EXECUTIONS)

def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP)

def workerFailures = []

currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24)
currentBuild.description = "${params.CI_GROUP}<br />Executions: ${params.NUMBER_EXECUTIONS}"

// Note: If you increase agent count, it will execute NUMBER_EXECUTIONS per agent. It will not divide them up amongst the agents
// e.g. NUMBER_EXECUTIONS = 25, agentCount = 4 results in 100 total executions
def agentCount = 1
currentBuild.description = "${params.CI_GROUP}<br />Agents: ${AGENT_COUNT}<br />Executions: ${params.NUMBER_EXECUTIONS}"

stage("Kibana Pipeline") {
timeout(time: 180, unit: 'MINUTES') {
timestamps {
ansiColor('xterm') {
def agents = [:]
for(def agentNumber = 1; agentNumber <= agentCount; agentNumber++) {
for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) {
def agentNumberInside = agentNumber
def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would this be better as a closure or method with a name?
No big deal, but names are quicker to grok, vs not being able to be lazy and having to read the code lol ahahahaa

agents["agent-${agentNumber}"] = {
catchError {
print "Agent ${agentNumberInside} - ${agentExecutions} executions"

kibanaPipeline.withWorkers('flaky-test-runner', {
if (!IS_XPACK) {
kibanaPipeline.buildOss()
Expand All @@ -37,7 +39,7 @@ stage("Kibana Pipeline") {
} else {
kibanaPipeline.buildXpack()
}
}, getWorkerMap(agentNumber, params.NUMBER_EXECUTIONS.toInteger(), worker, workerFailures))()
}, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
}
}
}
Expand Down Expand Up @@ -77,17 +79,20 @@ def getWorkerFromParams(isXpack, job, ciGroup) {
}
}

def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 14) {
def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkerProcesses = 12) {
def workerMap = [:]
def numberOfWorkers = Math.min(numberOfExecutions, maxWorkers)
def numberOfWorkers = Math.min(numberOfExecutions, maxWorkerProcesses)

for(def i = 1; i <= numberOfWorkers; i++) {
def workerExecutions = numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0)

workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber ->
for(def j = 0; j < workerExecutions; j++) {
print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}"
withEnv(["JOB=agent-${agentNumber}-worker-${workerNumber}-${j}"]) {
withEnv([
"JOB=agent-${agentNumber}-worker-${workerNumber}-${j}",
"REMOVE_KIBANA_INSTALL_DIR=1",
]) {
catchError {
try {
worker(workerNumber)
Expand All @@ -104,10 +109,23 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor
return workerMap
}

def getAgentCount(executions) {
// Increase agent count every 24 worker processess, up to 3 agents maximum
return Math.min(3, 1 + floor(executions/24))
}

def trunc(str, length) {
if (str.size() >= length) {
return str.take(length) + "..."
}

return str;
}

// All of the real rounding/truncating methods are sandboxed
def floor(num) {
return num
.toString()
.split('\\.')[0]
.toInteger()
}
8 changes: 1 addition & 7 deletions test/scripts/jenkins_ci_group.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,6 @@
#!/usr/bin/env bash

set -e

if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi

export TEST_BROWSER_HEADLESS=1
source test/scripts/jenkins_test_setup.sh

if [[ -z "$IS_PIPELINE_JOB" ]] ; then
yarn run grunt functionalTests:ensureAllTestsInCiGroup;
Expand Down
6 changes: 1 addition & 5 deletions test/scripts/jenkins_firefox_smoke.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
#!/usr/bin/env bash

set -e

if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
source test/scripts/jenkins_test_setup.sh

if [[ -z "$IS_PIPELINE_JOB" ]] ; then
node scripts/build --debug --oss;
Expand Down
20 changes: 20 additions & 0 deletions test/scripts/jenkins_test_setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
set -e

function post_work() {
set +e
if [[ -z "$IS_PIPELINE_JOB" ]] ; then
node "$KIBANA_DIR/scripts/report_failed_tests"
fi

if [[ -z "$REMOVE_KIBANA_INSTALL_DIR" && -z "$KIBANA_INSTALL_DIR" && -d "$KIBANA_INSTALL_DIR" ]]; then
rm -rf "$REMOVE_KIBANA_INSTALL_DIR"
fi
}

trap 'post_work' EXIT
wayneseymour marked this conversation as resolved.
Show resolved Hide resolved

export TEST_BROWSER_HEADLESS=1

if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
9 changes: 1 addition & 8 deletions test/scripts/jenkins_visual_regression.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,6 @@
#!/usr/bin/env bash

set -e

if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi

source test/scripts/jenkins_test_setup.sh
source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"

if [[ -z "$IS_PIPELINE_JOB" ]] ; then
Expand All @@ -22,8 +17,6 @@ else
export KIBANA_INSTALL_DIR="$destDir"
fi

export TEST_BROWSER_HEADLESS=1

checks-reporter-with-killswitch "Kibana visual regression tests" \
yarn run percy exec -t 500 \
node scripts/functional_tests \
Expand Down
8 changes: 1 addition & 7 deletions test/scripts/jenkins_xpack_ci_group.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,6 @@
#!/usr/bin/env bash

set -e

if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi

export TEST_BROWSER_HEADLESS=1
source test/scripts/jenkins_test_setup.sh

if [[ -z "$IS_PIPELINE_JOB" ]] ; then
echo " -> Ensuring all functional tests are in a ciGroup"
Expand Down
8 changes: 1 addition & 7 deletions test/scripts/jenkins_xpack_firefox_smoke.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
#!/usr/bin/env bash

set -e

if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi
source test/scripts/jenkins_test_setup.sh

if [[ -z "$IS_PIPELINE_JOB" ]] ; then
node scripts/build --debug --no-oss;
Expand All @@ -21,8 +17,6 @@ else
export KIBANA_INSTALL_DIR="$destDir"
fi

export TEST_BROWSER_HEADLESS=1

cd "$XPACK_DIR"

checks-reporter-with-killswitch "X-Pack firefox smoke test" \
Expand Down
9 changes: 1 addition & 8 deletions test/scripts/jenkins_xpack_visual_regression.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,6 @@
#!/usr/bin/env bash

set -e

if [[ -n "$IS_PIPELINE_JOB" ]] ; then
source src/dev/ci_setup/setup_env.sh
fi

source test/scripts/jenkins_test_setup.sh
source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"

if [[ -z "$IS_PIPELINE_JOB" ]] ; then
Expand All @@ -23,8 +18,6 @@ else
export KIBANA_INSTALL_DIR="$destDir"
fi

export TEST_BROWSER_HEADLESS=1

cd "$XPACK_DIR"

checks-reporter-with-killswitch "X-Pack visual regression tests" \
Expand Down