From e8ba96d9e12b06bb95253b7f385898d64a51c7f5 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Mon, 21 Oct 2019 16:40:37 -0400 Subject: [PATCH 1/5] Allocate multiple flaky test runner agents as executions grow --- .ci/Jenkinsfile_flaky | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.ci/Jenkinsfile_flaky b/.ci/Jenkinsfile_flaky index 3f77243da2c1b..9518136e10bfb 100644 --- a/.ci/Jenkinsfile_flaky +++ b/.ci/Jenkinsfile_flaky @@ -8,26 +8,28 @@ def JOB_PARTS = params.CI_GROUP.split(':') def IS_XPACK = JOB_PARTS[0] == 'xpack' def JOB = JOB_PARTS[1] def CI_GROUP = JOB_PARTS.size() > 2 ? JOB_PARTS[2] : '' +def EXECUTIONS = params.NUMBER_EXECUTIONS.toInteger() +def AGENT_COUNT = getAgentCount(EXECUTIONS) def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP) def workerFailures = [] currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24) -currentBuild.description = "${params.CI_GROUP}
Executions: ${params.NUMBER_EXECUTIONS}" - -// Note: If you increase agent count, it will execute NUMBER_EXECUTIONS per agent. It will not divide them up amongst the agents -// e.g. NUMBER_EXECUTIONS = 25, agentCount = 4 results in 100 total executions -def agentCount = 1 +currentBuild.description = "${params.CI_GROUP}
Agents: ${AGENT_COUNT}
Executions: ${params.NUMBER_EXECUTIONS}" stage("Kibana Pipeline") { timeout(time: 180, unit: 'MINUTES') { timestamps { ansiColor('xterm') { def agents = [:] - for(def agentNumber = 1; agentNumber <= agentCount; agentNumber++) { + for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) { + def agentNumberInside = agentNumber + def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0) agents["agent-${agentNumber}"] = { catchError { + print "Agent ${agentNumberInside} - ${agentExecutions} executions" + kibanaPipeline.withWorkers('flaky-test-runner', { if (!IS_XPACK) { kibanaPipeline.buildOss() @@ -37,7 +39,7 @@ stage("Kibana Pipeline") { } else { kibanaPipeline.buildXpack() } - }, getWorkerMap(agentNumber, params.NUMBER_EXECUTIONS.toInteger(), worker, workerFailures))() + }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))() } } } @@ -77,7 +79,7 @@ def getWorkerFromParams(isXpack, job, ciGroup) { } } -def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 14) { +def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 8) { def workerMap = [:] def numberOfWorkers = Math.min(numberOfExecutions, maxWorkers) @@ -104,6 +106,11 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor return workerMap } +def getAgentCount(executions) { + // Increase agent count every 40 workers, up to 3 agents maximum + return Math.min(3, 1 + floor(executions/40)) +} + def trunc(str, length) { if (str.size() >= length) { return str.take(length) + "..." @@ -111,3 +118,11 @@ def trunc(str, length) { return str; } + +// All of the real rounding/truncating methods are sandboxed +def floor(num) { + return num + .toString() + .split('\\.')[0] + .toInteger() +} From 435aed52cffc6b3d22e52debc4e477e87de8f5f5 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 22 Oct 2019 16:05:36 -0400 Subject: [PATCH 2/5] WIP support for deleting kibana install dir during CI --- test/scripts/jenkins_ci_group.sh | 2 ++ test/scripts/jenkins_firefox_smoke.sh | 2 ++ test/scripts/jenkins_post.sh | 3 +++ test/scripts/jenkins_visual_regression.sh | 2 ++ test/scripts/jenkins_xpack_ci_group.sh | 2 ++ test/scripts/jenkins_xpack_firefox_smoke.sh | 2 ++ test/scripts/jenkins_xpack_visual_regression.sh | 2 ++ 7 files changed, 15 insertions(+) create mode 100755 test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh index 274cca695b535..6d0a3af37d8f5 100755 --- a/test/scripts/jenkins_ci_group.sh +++ b/test/scripts/jenkins_ci_group.sh @@ -26,3 +26,5 @@ if [ "$CI_GROUP" == "1" ]; then yarn run grunt run:pluginFunctionalTestsRelease --from=source; yarn run grunt run:interpreterFunctionalTestsRelease; fi + +source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_firefox_smoke.sh b/test/scripts/jenkins_firefox_smoke.sh index 69cedc9657340..d11639b2f5c51 100755 --- a/test/scripts/jenkins_firefox_smoke.sh +++ b/test/scripts/jenkins_firefox_smoke.sh @@ -28,3 +28,5 @@ checks-reporter-with-killswitch "Firefox smoke test" \ --kibana-install-dir "$installDir" \ --include-tag "smoke" \ --config test/functional/config.firefox.js; + +source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_post.sh b/test/scripts/jenkins_post.sh new file mode 100755 index 0000000000000..5e10052a05024 --- /dev/null +++ b/test/scripts/jenkins_post.sh @@ -0,0 +1,3 @@ +if [[ -z "$REMOVE_KIBANA_INSTALL_DIR" && -z "$KIBANA_INSTALL_DIR" && -d "$KIBANA_INSTALL_DIR" ]]; then + rm -rf "$REMOVE_KIBANA_INSTALL_DIR" +fi diff --git a/test/scripts/jenkins_visual_regression.sh b/test/scripts/jenkins_visual_regression.sh index fdda24423d977..9137928c00dd0 100755 --- a/test/scripts/jenkins_visual_regression.sh +++ b/test/scripts/jenkins_visual_regression.sh @@ -30,3 +30,5 @@ checks-reporter-with-killswitch "Kibana visual regression tests" \ --debug --bail \ --kibana-install-dir "$installDir" \ --config test/visual_regression/config.ts; + +source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_xpack_ci_group.sh b/test/scripts/jenkins_xpack_ci_group.sh index 0b9666b33deca..24cc4c7453bd5 100755 --- a/test/scripts/jenkins_xpack_ci_group.sh +++ b/test/scripts/jenkins_xpack_ci_group.sh @@ -64,3 +64,5 @@ echo "" # --config "test/functional/config.firefox.js" # echo "" # echo "" + +source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_xpack_firefox_smoke.sh b/test/scripts/jenkins_xpack_firefox_smoke.sh index 18ed468de1317..6ca6fbd15c158 100755 --- a/test/scripts/jenkins_xpack_firefox_smoke.sh +++ b/test/scripts/jenkins_xpack_firefox_smoke.sh @@ -31,3 +31,5 @@ checks-reporter-with-killswitch "X-Pack firefox smoke test" \ --kibana-install-dir "$KIBANA_INSTALL_DIR" \ --include-tag "smoke" \ --config test/functional/config.firefox.js; + +source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_xpack_visual_regression.sh b/test/scripts/jenkins_xpack_visual_regression.sh index 0f3275d45f13b..e6e2d60875d86 100755 --- a/test/scripts/jenkins_xpack_visual_regression.sh +++ b/test/scripts/jenkins_xpack_visual_regression.sh @@ -33,3 +33,5 @@ checks-reporter-with-killswitch "X-Pack visual regression tests" \ --debug --bail \ --kibana-install-dir "$KIBANA_INSTALL_DIR" \ --config test/visual_regression/config.js; + +source test/scripts/jenkins_post.sh From 345d81bedd93cbe6ec13be21ec68690537601f61 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 23 Oct 2019 12:26:52 -0400 Subject: [PATCH 3/5] Add setup script for testing scripts --- test/scripts/jenkins_ci_group.sh | 10 +--------- test/scripts/jenkins_firefox_smoke.sh | 8 +------- test/scripts/jenkins_post.sh | 3 --- test/scripts/jenkins_test_setup.sh | 20 +++++++++++++++++++ test/scripts/jenkins_visual_regression.sh | 11 +--------- test/scripts/jenkins_xpack_ci_group.sh | 10 +--------- test/scripts/jenkins_xpack_firefox_smoke.sh | 10 +--------- .../jenkins_xpack_visual_regression.sh | 11 +--------- 8 files changed, 26 insertions(+), 57 deletions(-) delete mode 100755 test/scripts/jenkins_post.sh create mode 100644 test/scripts/jenkins_test_setup.sh diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh index 6d0a3af37d8f5..c6bddb69aa570 100755 --- a/test/scripts/jenkins_ci_group.sh +++ b/test/scripts/jenkins_ci_group.sh @@ -1,12 +1,6 @@ #!/usr/bin/env bash -set -e - -if [[ -n "$IS_PIPELINE_JOB" ]] ; then - source src/dev/ci_setup/setup_env.sh -fi - -export TEST_BROWSER_HEADLESS=1 +source test/scripts/jenkins_test_setup.sh if [[ -z "$IS_PIPELINE_JOB" ]] ; then yarn run grunt functionalTests:ensureAllTestsInCiGroup; @@ -26,5 +20,3 @@ if [ "$CI_GROUP" == "1" ]; then yarn run grunt run:pluginFunctionalTestsRelease --from=source; yarn run grunt run:interpreterFunctionalTestsRelease; fi - -source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_firefox_smoke.sh b/test/scripts/jenkins_firefox_smoke.sh index d11639b2f5c51..9a31f5f43d224 100755 --- a/test/scripts/jenkins_firefox_smoke.sh +++ b/test/scripts/jenkins_firefox_smoke.sh @@ -1,10 +1,6 @@ #!/usr/bin/env bash -set -e - -if [[ -n "$IS_PIPELINE_JOB" ]] ; then - source src/dev/ci_setup/setup_env.sh -fi +source test/scripts/jenkins_test_setup.sh if [[ -z "$IS_PIPELINE_JOB" ]] ; then node scripts/build --debug --oss; @@ -28,5 +24,3 @@ checks-reporter-with-killswitch "Firefox smoke test" \ --kibana-install-dir "$installDir" \ --include-tag "smoke" \ --config test/functional/config.firefox.js; - -source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_post.sh b/test/scripts/jenkins_post.sh deleted file mode 100755 index 5e10052a05024..0000000000000 --- a/test/scripts/jenkins_post.sh +++ /dev/null @@ -1,3 +0,0 @@ -if [[ -z "$REMOVE_KIBANA_INSTALL_DIR" && -z "$KIBANA_INSTALL_DIR" && -d "$KIBANA_INSTALL_DIR" ]]; then - rm -rf "$REMOVE_KIBANA_INSTALL_DIR" -fi diff --git a/test/scripts/jenkins_test_setup.sh b/test/scripts/jenkins_test_setup.sh new file mode 100644 index 0000000000000..e2dd0bc276bb6 --- /dev/null +++ b/test/scripts/jenkins_test_setup.sh @@ -0,0 +1,20 @@ +set -e + +function post_work() { + set +e + if [[ -z "$IS_PIPELINE_JOB" ]] ; then + node "$KIBANA_DIR/scripts/report_failed_tests" + fi + + if [[ -z "$REMOVE_KIBANA_INSTALL_DIR" && -z "$KIBANA_INSTALL_DIR" && -d "$KIBANA_INSTALL_DIR" ]]; then + rm -rf "$REMOVE_KIBANA_INSTALL_DIR" + fi +} + +trap 'post_work' EXIT + +export TEST_BROWSER_HEADLESS=1 + +if [[ -n "$IS_PIPELINE_JOB" ]] ; then + source src/dev/ci_setup/setup_env.sh +fi diff --git a/test/scripts/jenkins_visual_regression.sh b/test/scripts/jenkins_visual_regression.sh index 9137928c00dd0..9ca1c0f08d2c9 100755 --- a/test/scripts/jenkins_visual_regression.sh +++ b/test/scripts/jenkins_visual_regression.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash -set -e - -if [[ -n "$IS_PIPELINE_JOB" ]] ; then - source src/dev/ci_setup/setup_env.sh -fi - +source test/scripts/jenkins_test_setup.sh source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh" if [[ -z "$IS_PIPELINE_JOB" ]] ; then @@ -22,13 +17,9 @@ else export KIBANA_INSTALL_DIR="$destDir" fi -export TEST_BROWSER_HEADLESS=1 - checks-reporter-with-killswitch "Kibana visual regression tests" \ yarn run percy exec -t 500 \ node scripts/functional_tests \ --debug --bail \ --kibana-install-dir "$installDir" \ --config test/visual_regression/config.ts; - -source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_xpack_ci_group.sh b/test/scripts/jenkins_xpack_ci_group.sh index 24cc4c7453bd5..fba05f8f252d7 100755 --- a/test/scripts/jenkins_xpack_ci_group.sh +++ b/test/scripts/jenkins_xpack_ci_group.sh @@ -1,12 +1,6 @@ #!/usr/bin/env bash -set -e - -if [[ -n "$IS_PIPELINE_JOB" ]] ; then - source src/dev/ci_setup/setup_env.sh -fi - -export TEST_BROWSER_HEADLESS=1 +source test/scripts/jenkins_test_setup.sh if [[ -z "$IS_PIPELINE_JOB" ]] ; then echo " -> Ensuring all functional tests are in a ciGroup" @@ -64,5 +58,3 @@ echo "" # --config "test/functional/config.firefox.js" # echo "" # echo "" - -source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_xpack_firefox_smoke.sh b/test/scripts/jenkins_xpack_firefox_smoke.sh index 6ca6fbd15c158..43220459bcb97 100755 --- a/test/scripts/jenkins_xpack_firefox_smoke.sh +++ b/test/scripts/jenkins_xpack_firefox_smoke.sh @@ -1,10 +1,6 @@ #!/usr/bin/env bash -set -e - -if [[ -n "$IS_PIPELINE_JOB" ]] ; then - source src/dev/ci_setup/setup_env.sh -fi +source test/scripts/jenkins_test_setup.sh if [[ -z "$IS_PIPELINE_JOB" ]] ; then node scripts/build --debug --no-oss; @@ -21,8 +17,6 @@ else export KIBANA_INSTALL_DIR="$destDir" fi -export TEST_BROWSER_HEADLESS=1 - cd "$XPACK_DIR" checks-reporter-with-killswitch "X-Pack firefox smoke test" \ @@ -31,5 +25,3 @@ checks-reporter-with-killswitch "X-Pack firefox smoke test" \ --kibana-install-dir "$KIBANA_INSTALL_DIR" \ --include-tag "smoke" \ --config test/functional/config.firefox.js; - -source test/scripts/jenkins_post.sh diff --git a/test/scripts/jenkins_xpack_visual_regression.sh b/test/scripts/jenkins_xpack_visual_regression.sh index e6e2d60875d86..5699f9e5ee7c1 100755 --- a/test/scripts/jenkins_xpack_visual_regression.sh +++ b/test/scripts/jenkins_xpack_visual_regression.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash -set -e - -if [[ -n "$IS_PIPELINE_JOB" ]] ; then - source src/dev/ci_setup/setup_env.sh -fi - +source test/scripts/jenkins_test_setup.sh source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh" if [[ -z "$IS_PIPELINE_JOB" ]] ; then @@ -23,8 +18,6 @@ else export KIBANA_INSTALL_DIR="$destDir" fi -export TEST_BROWSER_HEADLESS=1 - cd "$XPACK_DIR" checks-reporter-with-killswitch "X-Pack visual regression tests" \ @@ -33,5 +26,3 @@ checks-reporter-with-killswitch "X-Pack visual regression tests" \ --debug --bail \ --kibana-install-dir "$KIBANA_INSTALL_DIR" \ --config test/visual_regression/config.js; - -source test/scripts/jenkins_post.sh From 9823e6380a601ccf0720430e51e0c1d4d5645c3b Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 23 Oct 2019 15:35:09 -0400 Subject: [PATCH 4/5] Add REMOVE_KIBANA_INSTALL_DIR=1 to flaky test runner --- .ci/Jenkinsfile_flaky | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile_flaky b/.ci/Jenkinsfile_flaky index 9518136e10bfb..67807353a7b01 100644 --- a/.ci/Jenkinsfile_flaky +++ b/.ci/Jenkinsfile_flaky @@ -89,7 +89,10 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber -> for(def j = 0; j < workerExecutions; j++) { print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}" - withEnv(["JOB=agent-${agentNumber}-worker-${workerNumber}-${j}"]) { + withEnv([ + "JOB=agent-${agentNumber}-worker-${workerNumber}-${j}", + "REMOVE_KIBANA_INSTALL_DIR=1", + ]) { catchError { try { worker(workerNumber) From 5d12ed9343be8a1fa59bdaee6b26ab4492829b88 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Tue, 29 Oct 2019 14:01:40 -0400 Subject: [PATCH 5/5] Change flaky test runner worker processes from 8 to 12 --- .ci/Jenkinsfile_flaky | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile_flaky b/.ci/Jenkinsfile_flaky index 67807353a7b01..e1cbac0528b1f 100644 --- a/.ci/Jenkinsfile_flaky +++ b/.ci/Jenkinsfile_flaky @@ -79,9 +79,9 @@ def getWorkerFromParams(isXpack, job, ciGroup) { } } -def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 8) { +def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkerProcesses = 12) { def workerMap = [:] - def numberOfWorkers = Math.min(numberOfExecutions, maxWorkers) + def numberOfWorkers = Math.min(numberOfExecutions, maxWorkerProcesses) for(def i = 1; i <= numberOfWorkers; i++) { def workerExecutions = numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0) @@ -110,8 +110,8 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor } def getAgentCount(executions) { - // Increase agent count every 40 workers, up to 3 agents maximum - return Math.min(3, 1 + floor(executions/40)) + // Increase agent count every 24 worker processess, up to 3 agents maximum + return Math.min(3, 1 + floor(executions/24)) } def trunc(str, length) {