From d75c9e78dd2592cb69a6f8016c32609052dd67ab Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 8 Mar 2018 13:03:14 +0100 Subject: [PATCH 1/8] Run tests on multiple clusters in parallel --- Jenkinsfile.groovy | 71 +++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 3e383bfd7..4e2a24360 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -19,6 +19,43 @@ def notifySlack(String buildStatus = 'STARTED') { slackSend(color: color, channel: '#status-k8s', message: msg) } +def kubeConfigRoot = "/home/jenkins/.kube/" + +def buildTestSteps(String kubeconfig) { + return { + timestamps { + lock("${kubeconfig}-${params.TESTNAMESPACE}-${env.GIT_COMMIT}") { + withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { + withEnv([ + "ENTERPRISEIMAGE=${params.ENTERPRISEIMAGE}", + "IMAGETAG=${env.GIT_COMMIT}", + "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", + "LONG=${params.LONG ? 1 : 0}", + "PUSHIMAGES=1", + "TESTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", + ]) { + sh "make run-tests" + } + } + } + } + } +} + +def buildCleanupSteps(String kubeconfig) { + return { + timestamps { + withEnv([ + "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", + "TESTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", + ]) { + sh "make cleanup-tests" + } + } + } + } +} + pipeline { options { buildDiscarder(logRotator(daysToKeepStr: '7', numToKeepStr: '10')) @@ -26,7 +63,7 @@ pipeline { agent any parameters { booleanParam(name: 'LONG', defaultValue: false, description: 'Execute long running tests') - string(name: 'KUBECONFIG', defaultValue: '/home/jenkins/.kube/scw-183a3b', description: 'KUBECONFIG controls which k8s cluster is used', ) + string(name: 'KUBECONFIGS', defaultValue: 'scw-183a3b,c11', description: 'KUBECONFIGS is a comma separated list of Kubernetes configuration files (relative to /home/jenkins/.kube) on which the tests are run', ) string(name: 'TESTNAMESPACE', defaultValue: 'jenkins', description: 'TESTNAMESPACE sets the kubernetes namespace to ru tests in (this must be short!!)', ) string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', ) } @@ -45,36 +82,24 @@ pipeline { } stage('Test') { steps { - timestamps { - lock("${params.TESTNAMESPACE}-${env.GIT_COMMIT}") { - withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { - withEnv([ - "ENTERPRISEIMAGE=${params.ENTERPRISEIMAGE}", - "IMAGETAG=${env.GIT_COMMIT}", - "KUBECONFIG=${params.KUBECONFIG}", - "LONG=${params.LONG ? 1 : 0}", - "PUSHIMAGES=1", - "TESTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", - ]) { - sh "make run-tests" - } - } - } + def configs = "{params.KUBECONFIGS}".split(",") + def testTasks[:] + for (kubeconfig in configs) { + testTasks["${kubeconfig}"] = buildTestSteps(kubeconfig) } + parallel testTasks } } } post { always { - timestamps { - withEnv([ - "KUBECONFIG=${params.KUBECONFIG}", - "TESTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", - ]) { - sh "make cleanup-tests" - } + def configs = "{params.KUBECONFIGS}".split(",") + def cleanupTasks[:] + for (kubeconfig in configs) { + cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeconfig) } + parallel cleanupTasks } failure { notifySlack('FAILURE') From 857c0753e2db14ca154d5ecc6a1ef4e713936c45 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 8 Mar 2018 13:07:42 +0100 Subject: [PATCH 2/8] Fixed indent --- Jenkinsfile.groovy | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 4e2a24360..c9e70aed7 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -45,12 +45,11 @@ def buildTestSteps(String kubeconfig) { def buildCleanupSteps(String kubeconfig) { return { timestamps { - withEnv([ - "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", - "TESTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", - ]) { - sh "make cleanup-tests" - } + withEnv([ + "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", + "TESTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", + ]) { + sh "make cleanup-tests" } } } From cfa7bc760c5fef2937d71783ce7c1118193aa1f6 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 8 Mar 2018 13:08:47 +0100 Subject: [PATCH 3/8] Fixed syntax --- Jenkinsfile.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index c9e70aed7..15dbb38ef 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -82,7 +82,7 @@ pipeline { stage('Test') { steps { def configs = "{params.KUBECONFIGS}".split(",") - def testTasks[:] + def testTasks = [:] for (kubeconfig in configs) { testTasks["${kubeconfig}"] = buildTestSteps(kubeconfig) } @@ -94,7 +94,7 @@ pipeline { post { always { def configs = "{params.KUBECONFIGS}".split(",") - def cleanupTasks[:] + def cleanupTasks = [:] for (kubeconfig in configs) { cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeconfig) } From 9c8632320d608a8eb74abbb36b92e0b8c7d4defa Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 8 Mar 2018 13:10:25 +0100 Subject: [PATCH 4/8] Fixing step (i hope) --- Jenkinsfile.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 15dbb38ef..a4d967574 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -80,14 +80,14 @@ pipeline { } } stage('Test') { - steps { + //steps { def configs = "{params.KUBECONFIGS}".split(",") def testTasks = [:] for (kubeconfig in configs) { testTasks["${kubeconfig}"] = buildTestSteps(kubeconfig) } parallel testTasks - } + //} } } From 642f9bd3f33cb44c9a612b88bfdce4ade19b517b Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 8 Mar 2018 13:11:43 +0100 Subject: [PATCH 5/8] Wrapping in script --- Jenkinsfile.groovy | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index a4d967574..3645461d4 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -80,25 +80,29 @@ pipeline { } } stage('Test') { - //steps { - def configs = "{params.KUBECONFIGS}".split(",") - def testTasks = [:] - for (kubeconfig in configs) { - testTasks["${kubeconfig}"] = buildTestSteps(kubeconfig) + steps { + script { + def configs = "{params.KUBECONFIGS}".split(",") + def testTasks = [:] + for (kubeconfig in configs) { + testTasks["${kubeconfig}"] = buildTestSteps(kubeconfig) + } + parallel testTasks } - parallel testTasks - //} + } } } post { always { - def configs = "{params.KUBECONFIGS}".split(",") - def cleanupTasks = [:] - for (kubeconfig in configs) { - cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeconfig) + script { + def configs = "{params.KUBECONFIGS}".split(",") + def cleanupTasks = [:] + for (kubeconfig in configs) { + cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeconfig) + } + parallel cleanupTasks } - parallel cleanupTasks } failure { notifySlack('FAILURE') From 52731f573578a86b4b78c85d1b893084cb9edd29 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 8 Mar 2018 13:14:30 +0100 Subject: [PATCH 6/8] Fix $ --- Jenkinsfile.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 3645461d4..a83faaf45 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -82,7 +82,7 @@ pipeline { stage('Test') { steps { script { - def configs = "{params.KUBECONFIGS}".split(",") + def configs = "${params.KUBECONFIGS}".split(",") def testTasks = [:] for (kubeconfig in configs) { testTasks["${kubeconfig}"] = buildTestSteps(kubeconfig) @@ -96,7 +96,7 @@ pipeline { post { always { script { - def configs = "{params.KUBECONFIGS}".split(",") + def configs = "${params.KUBECONFIGS}".split(",") def cleanupTasks = [:] for (kubeconfig in configs) { cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeconfig) From 8942d2e2c823732094184da27cead22711a1c83d Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 8 Mar 2018 13:19:27 +0100 Subject: [PATCH 7/8] Bring kubeConfigRoot in scope --- Jenkinsfile.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index a83faaf45..bddda7759 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -19,9 +19,9 @@ def notifySlack(String buildStatus = 'STARTED') { slackSend(color: color, channel: '#status-k8s', message: msg) } -def kubeConfigRoot = "/home/jenkins/.kube/" +def kubeConfigRoot = "/home/jenkins/.kube" -def buildTestSteps(String kubeconfig) { +def buildTestSteps(String kubeConfigRoot, String kubeconfig) { return { timestamps { lock("${kubeconfig}-${params.TESTNAMESPACE}-${env.GIT_COMMIT}") { @@ -42,7 +42,7 @@ def buildTestSteps(String kubeconfig) { } } -def buildCleanupSteps(String kubeconfig) { +def buildCleanupSteps(String kubeConfigRoot, String kubeconfig) { return { timestamps { withEnv([ @@ -85,7 +85,7 @@ pipeline { def configs = "${params.KUBECONFIGS}".split(",") def testTasks = [:] for (kubeconfig in configs) { - testTasks["${kubeconfig}"] = buildTestSteps(kubeconfig) + testTasks["${kubeconfig}"] = buildTestSteps(kubeConfigRoot, kubeconfig) } parallel testTasks } @@ -99,7 +99,7 @@ pipeline { def configs = "${params.KUBECONFIGS}".split(",") def cleanupTasks = [:] for (kubeconfig in configs) { - cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeconfig) + cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeConfigRoot, kubeconfig) } parallel cleanupTasks } From 29a69e94bcf83a00794fe08578977332a49bd7b8 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 8 Mar 2018 13:30:42 +0100 Subject: [PATCH 8/8] Build test in build step --- Jenkinsfile.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index bddda7759..c16ad7357 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -72,9 +72,12 @@ pipeline { timestamps { withEnv([ "IMAGETAG=${env.GIT_COMMIT}", + "LONG=${params.LONG ? 1 : 0}", + "PUSHIMAGES=1", ]) { sh "make" sh "make run-unit-tests" + sh "make docker-test" } } }