From ae7c20dea02a77a467b1fc4eac88343202fc2f4e Mon Sep 17 00:00:00 2001 From: Devyn Goetsch Date: Sat, 9 Feb 2019 19:57:12 -0600 Subject: [PATCH 1/2] Add declarative pipeline harness --- .../unit/DeclarativePipelineHarness.groovy | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 src/main/groovy/com/lesfurets/jenkins/unit/DeclarativePipelineHarness.groovy diff --git a/src/main/groovy/com/lesfurets/jenkins/unit/DeclarativePipelineHarness.groovy b/src/main/groovy/com/lesfurets/jenkins/unit/DeclarativePipelineHarness.groovy new file mode 100644 index 00000000..c8038d1a --- /dev/null +++ b/src/main/groovy/com/lesfurets/jenkins/unit/DeclarativePipelineHarness.groovy @@ -0,0 +1,164 @@ +package com.lesfurets.jenkins.unit + +class DeclarativePipelineHarness { + BasePipelineTest base + + Map callStacks = [:] + Map> identifiedCallStacks = [:] + + DeclarativePipelineHarness(base) { + this.base = base + registerCallStack('pipeline') + registerCallStack('options') + registerCallStack('parameters') + registerCallStack('environment') + registerCallStack('triggers') + registerCallStack('tools') + registerCallStack('input') + + registerCallStack('agent') + registerCallStack('docker') + registerCallStack('dockerfile') + registerCallStack('node') + + registerCallStack('stages') + registerCallStack('parallel') + registerCallStack('script') + registerCallStack('steps') + + registerCallStack('when') + registerCallStack('allOf') + registerCallStack('anyOf') + registerCallStack('expression') + + //Register post methods + registerCallStack('post') + registerCallStack('always') + registerCallStack('cleanup') + registerCallStack('success') + registerCallStack('failure') + registerCallStack('regression') + registerCallStack('changed') + registerCallStack('fixed') + registerCallStack('aborted') + registerCallStack('unstable') + registerCallStack('unsuccessful') + + registerIdentifiedCallStack('stage', String.class) + registerIdentifiedCallStack('node', String.class) + registerIdentifiedCallStack('withCredentials', Object.class) + registerIdentifiedCallStack('withEnv', List.class) + registerIdentifiedCallStack('dir', String.class) + } + + void runCall(String name, Integer idx = 0, Boolean doClear = true) { + CallStack callStack = callStacks[name] + if(callStack== null) { + throw new RuntimeException("No such call stack ${name}") + } + callStack.run(idx, doClear) + } + + public void runIdentified(String name, T value, Boolean doClear = true) { + IdentifiedCallStack callStack = identifiedCallStacks[name] + if(callStack== null) {'pipeline' + throw new RuntimeException("No such call stack ${name}") + } + callStack.run(value, doClear) + } + + Object runIdentifiedByIndex(String name, Integer idx = 0, Boolean doClear = true) { + IdentifiedCallStack callStack = identifiedCallStacks[name] + if(callStack== null) { + throw new RuntimeException("No such call stack ${name}") + } + return callStack.run(idx, doClear) + } + + void registerCallStack(String name) { + callStacks.put(name, new CallStack(name, base.helper)) + } + + public void registerIdentifiedCallStack(String name, Class clazz) { + identifiedCallStacks.put(name, new IdentifiedCallStack(name, base.helper, clazz)) + } + + void clearAllCalls() { + base.helper.clearCallStack() + callStacks.values().forEach { it.clearCalls() } + identifiedCallStacks.values().forEach { it.clearCalls() } + } + + + class CallStack { + String name + PipelineTestHelper helper + + List calls = [] + + CallStack(name, helper) { + this.name = name + this.helper = helper + + this.helper.registerAllowedMethod(name, [Closure.class], { calls.add(it) }) + } + + void clearCalls() { + calls.clear() + } + + void run(Integer idx = 0, doClear = true) { + if(! (idx < calls.size())) { + throw new RuntimeException("not enough calls in stack for ${name}, expected idx: ${idx}, size: ${idx}") + } + def c = calls[idx] + if(doClear) { clearAllCalls() } + c() + } + } + + class IdentifiedCallStack { + String name + PipelineTestHelper helper + Class clazz + + List values = [] + List calls = [] + IdentifiedCallStack(name, helper, clazz) { + this.name = name + this.helper = helper + this.clazz = clazz + + this.helper.registerAllowedMethod(name, [clazz, Closure.class], { v, c -> + values.add(v) + calls.add(c) + }) + } + + void clearCalls() { + values.clear() + calls.clear() + } + + T run(Integer idx = 0, doClear = true) { + if(! (idx < calls.size())) { + throw new RuntimeException("not enough calls in stack for ${name}, expected idx: ${idx}, size: ${idx}") + } + + def c = calls[idx] + if(doClear) { clearAllCalls() } + c() + return values[idx] + } + + void run(T value, doClear = true) { + def idx = values.findIndexOf { it == value } + if(idx < 0) { + throw new RuntimeException("call for $value does not exist, existing values are ${values}") + } + def c = calls[idx] + if(doClear) { clearAllCalls() } + c() + } + } +} From cd84f14d507c0f3fa56128c42b6031957d64d738 Mon Sep 17 00:00:00 2001 From: Devyn Goetsch Date: Mon, 25 Mar 2019 14:21:14 -0500 Subject: [PATCH 2/2] Add unit test and basedeclarativepipelinetest --- README.md | 72 +++++++---- .../unit/BaseDeclarativePipelineTest.groovy | 77 ++++++++++++ .../unit/DeclarativePipelineHarness.groovy | 113 ++++++++---------- .../TestBaseDeclarativePipelineTest.groovy | 53 ++++++++ src/test/jenkins/job/declarative.jenkins | 23 ++++ 5 files changed, 253 insertions(+), 85 deletions(-) create mode 100644 src/main/groovy/com/lesfurets/jenkins/unit/BaseDeclarativePipelineTest.groovy create mode 100644 src/test/groovy/com/lesfurets/jenkins/TestBaseDeclarativePipelineTest.groovy create mode 100644 src/test/jenkins/job/declarative.jenkins diff --git a/README.md b/README.md index b68575e0..bf3f135d 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ You can mock built-in Jenkins commands, job configurations, see the stacktrace o ### Add to your project as test dependency -Maven: +Maven: ```xml @@ -40,7 +40,7 @@ You can write your tests in Groovy or Java 8, using the test framework you prefe The easiest entry point is extending the abstract class `BasePipelineTest`, which initializes the framework with JUnit. Let's say you wrote this awesome pipeline script, which builds and tests your project : - + ```groovy def execute() { node() { @@ -74,9 +74,9 @@ Now using the Jenkins Pipeline Unit you can unit test if it does the job : import com.lesfurets.jenkins.unit.BasePipelineTest class TestExampleJob extends BasePipelineTest { - + //... - + @Test void should_execute_without_errors() throws Exception { def script = loadScript("job/exampleJob.jenkins") @@ -117,7 +117,7 @@ You can define both environment variables and job execution parameters. @Before void setUp() throws Exception { super.setUp() - // Assigns false to a job parameter ENABLE_TEST_STAGE + // Assigns false to a job parameter ENABLE_TEST_STAGE binding.setVariable('ENABLE_TEST_STAGE', 'false') // Defines the previous execution status binding.getVariable('currentBuild').previousBuild = [result: 'UNSTABLE'] @@ -125,7 +125,7 @@ You can define both environment variables and job execution parameters. ``` The test helper already provides basic variables such as a very simple currentBuild definition. -You can redefine them as you wish. +You can redefine them as you wish. ### Mock Jenkins commands @@ -145,8 +145,8 @@ You can register interceptors to mock Jenkins commands, which may or may not ret } ``` -The test helper already includes some mocks, but the list is far from complete. -You need to _register allowed methods_ if you want to override these mocks and add others. +The test helper already includes some mocks, but the list is far from complete. +You need to _register allowed methods_ if you want to override these mocks and add others. Note that you need to provide a method signature and a callback (closure or lambda) in order to allow a method. Any method call which is not recognized will throw an exception. @@ -174,7 +174,7 @@ void should_execute_without_errors() throws Exception { ``` -This will check as well `mvn verify` has been called during the job execution. +This will check as well `mvn verify` has been called during the job execution. ### Check Pipeline status @@ -194,9 +194,9 @@ To verify your pipeline is failing you need to check the status with `BasePipeli class TestCase extends BasePipelineTest { @Test void check_build_status() throws Exception { - helper.registerAllowedMethod("sh", [String.class], {cmd-> + helper.registerAllowedMethod("sh", [String.class], {cmd-> // cmd.contains is helpfull to filter sh call which should fail the pipeline - if (cmd.contains("make")) { + if (cmd.contains("make")) { binding.getVariable('currentBuild').result = 'FAILURE' } }) @@ -258,7 +258,7 @@ You then can go ahead and commit this change in your SCM to check in the change. ## Configuration -The abstract class `BasePipelineTest` configures the helper with useful conventions: +The abstract class `BasePipelineTest` configures the helper with useful conventions: - It looks for pipeline scripts in your project in root (`./.`) and `src/main/jenkins` paths. - Jenkins pipelines let you load other scripts from a parent script with `load` command. @@ -282,7 +282,7 @@ class TestExampleJob extends BasePipelineTest { scriptExtension = 'pipeline' super.setUp() } - + } ``` @@ -302,12 +302,12 @@ This will work fine for such a project structure: ## Testing Shared Libraries -With [Shared Libraries](https://jenkins.io/doc/book/pipeline/shared-libraries/) Jenkins lets you share common code +With [Shared Libraries](https://jenkins.io/doc/book/pipeline/shared-libraries/) Jenkins lets you share common code on pipelines across different repositories of your organization. -Shared libraries are configured via a settings interface in Jenkins and imported +Shared libraries are configured via a settings interface in Jenkins and imported with `@Library` annotation in your scripts. -Testing pipeline scripts using external libraries is not trivial because the shared library code +Testing pipeline scripts using external libraries is not trivial because the shared library code is checked in another repository. JenkinsPipelineUnit lets you test shared libraries and pipelines depending on these libraries. @@ -349,23 +349,23 @@ Now let's test it: .implicit(false) .build() helper.registerSharedLibrary(library) - + runScript("job/library/exampleJob.jenkins") printCallStack() ``` Notice how we defined the shared library and registered it to the helper. -Library definition is done via a fluent API which lets you set the same configurations as in +Library definition is done via a fluent API which lets you set the same configurations as in [Jenkins Global Pipeline Libraries](https://jenkins.io/doc/book/pipeline/shared-libraries/#using-libraries). The `retriever` and `targetPath` fields tell the framework how to fetch the sources of the library, in which local path. The framework comes with two naive but useful retrievers, `gitSource` and `localSource`. You can write your own retriever by implementing the `SourceRetriever` interface. -Note that properties `defaultVersion`, `allowOverride` and `implicit` are optional with +Note that properties `defaultVersion`, `allowOverride` and `implicit` are optional with default values `master`, `true` and `false`. -Now if we execute this test, the framework will fetch the sources from the Git repository and +Now if we execute this test, the framework will fetch the sources from the Git repository and load classes, scripts, global variables and resources found in the library. The callstack of this execution will look like the following: @@ -385,24 +385,46 @@ libraryJob.run() libraryJob.sh(curl -H 'Content-Type: application/json' -X POST -d '{"name" : "Ben"}' http://acme.com) ``` +## Declarative Pipeline Support + +Declarative pipelines are supported via a test harness. The tests use closures in the file as units of isolation. Effectively, +your tests describe which part of the jenkinsfile your test is verifying and don't interract with the rest of the file. Here is an example that +verifies that sh was invoked during the "init" stage. + +``` +@Test +void runAStage() { + super.jenkinsfile.run() + harness + .runStage('init') + .runClosureStep('steps') + + def call = helper.callStack.find { it.methodName == 'sh' } + assertThat(call.args.toList()).isEqualTo(['echo \'hello world\'']) +} +``` + +There is [a unit test that serves as a complete example](src/test/groovy/com/lesfurets/jenkins/TestBaseDeclarativePipelineTest.groovy). + + ## Note on CPS If you already fiddled with Jenkins pipeline DSL, you experienced strange errors during execution on Jenkins. This is because Jenkins does not directly execute your pipeline in Groovy, but transforms the pipeline code into an intermediate format to in order to run Groovy code in [Continuation Passing Style](https://en.wikipedia.org/wiki/Continuation-passing_style) (CPS). - -The usual errors are partly due to the -[the sandboxing Jenkins applies](https://wiki.jenkins-ci.org/display/JENKINS/Script+Security+Plugin#ScriptSecurityPlugin-GroovySandboxing) + +The usual errors are partly due to the +[the sandboxing Jenkins applies](https://wiki.jenkins-ci.org/display/JENKINS/Script+Security+Plugin#ScriptSecurityPlugin-GroovySandboxing) for security reasons, and partly due to the [serializability Jenkins imposes](https://github.com/jenkinsci/pipeline-plugin/blob/master/TUTORIAL.md#serializing-local-variables). Jenkins requires that at each execution step, the whole script context is serializable, in order to stop and resume the job execution. -To simulate this aspect, CPS versions of the helpers transform your scripts into the CPS format and check if at each step your script context is serializable. +To simulate this aspect, CPS versions of the helpers transform your scripts into the CPS format and check if at each step your script context is serializable. To use this _*experimental*_ feature, you can use the abstract class `BasePipelineTestCPS` instead of `BasePipelineTest`. You may see some changes in the call stacks that the helper registers. -Note also that the serialization used to test is not the same as what Jenkins uses. +Note also that the serialization used to test is not the same as what Jenkins uses. You may find some incoherence in that level. ## Contributing diff --git a/src/main/groovy/com/lesfurets/jenkins/unit/BaseDeclarativePipelineTest.groovy b/src/main/groovy/com/lesfurets/jenkins/unit/BaseDeclarativePipelineTest.groovy new file mode 100644 index 00000000..b5a06b33 --- /dev/null +++ b/src/main/groovy/com/lesfurets/jenkins/unit/BaseDeclarativePipelineTest.groovy @@ -0,0 +1,77 @@ +package com.lesfurets.jenkins.unit + +abstract class BaseDeclarativePipelineTest extends BasePipelineTest { + + Script jenkinsfile = null + + DeclarativePipelineHarness harness + + + @Override + void setUp() { + if(harness == null || jenkinsfile == null) { + super.setUp() + harness = new DeclarativePipelineHarness(this, allowedClosureSteps(), allowedParameterizedClosureSteps()) + loadJenkinsfile() + } else { + harness.clearAllCalls() + } + } + + abstract String jenkinsfileName() + + def loadJenkinsfile() { + this.jenkinsfile = super.loadScript(jenkinsfileName()) + } + + List defaultAllowedClosureSteps = ['pipeline', + 'options', + 'parameters', + 'environment', + 'triggers', + 'tools', + 'input', + + 'agent', + 'docker', + 'dockerfile', + 'node', + + 'stages', + 'parallel', + 'script', + 'steps', + + 'when', + 'allOf', + 'anyOf', + 'expression', + + 'post', + 'always', + 'cleanup', + 'success', + 'failure', + 'regression', + 'changed', + 'fixed', + 'aborted', + 'unstable', + 'unsuccessful'] + + Map> defaultAllowedParameterizedClosureSteps = [ + 'stage': String.class as Class, + 'node' : String.class as Class, + 'withCredentials': Object.class, + 'withEnv': List.class as Class, + 'dir': String.class as Class, + ] + + List allowedClosureSteps() { + return defaultAllowedClosureSteps + } + + Map> allowedParameterizedClosureSteps() { + return defaultAllowedParameterizedClosureSteps + } +} diff --git a/src/main/groovy/com/lesfurets/jenkins/unit/DeclarativePipelineHarness.groovy b/src/main/groovy/com/lesfurets/jenkins/unit/DeclarativePipelineHarness.groovy index c8038d1a..c717af1f 100644 --- a/src/main/groovy/com/lesfurets/jenkins/unit/DeclarativePipelineHarness.groovy +++ b/src/main/groovy/com/lesfurets/jenkins/unit/DeclarativePipelineHarness.groovy @@ -3,90 +3,82 @@ package com.lesfurets.jenkins.unit class DeclarativePipelineHarness { BasePipelineTest base - Map callStacks = [:] - Map> identifiedCallStacks = [:] + Map stepCallStacks = [:] + Map> parameterizedStepCallStacks = [:] - DeclarativePipelineHarness(base) { + Object lastClosureStepParameter = null + + List allowedClosureSteps + Map> allowedParameterizedClosureSteps + + DeclarativePipelineHarness(BasePipelineTest base, List closureSteps, Map> parameterizedClosureSteps) { this.base = base - registerCallStack('pipeline') - registerCallStack('options') - registerCallStack('parameters') - registerCallStack('environment') - registerCallStack('triggers') - registerCallStack('tools') - registerCallStack('input') - - registerCallStack('agent') - registerCallStack('docker') - registerCallStack('dockerfile') - registerCallStack('node') - - registerCallStack('stages') - registerCallStack('parallel') - registerCallStack('script') - registerCallStack('steps') - - registerCallStack('when') - registerCallStack('allOf') - registerCallStack('anyOf') - registerCallStack('expression') - - //Register post methods - registerCallStack('post') - registerCallStack('always') - registerCallStack('cleanup') - registerCallStack('success') - registerCallStack('failure') - registerCallStack('regression') - registerCallStack('changed') - registerCallStack('fixed') - registerCallStack('aborted') - registerCallStack('unstable') - registerCallStack('unsuccessful') - - registerIdentifiedCallStack('stage', String.class) - registerIdentifiedCallStack('node', String.class) - registerIdentifiedCallStack('withCredentials', Object.class) - registerIdentifiedCallStack('withEnv', List.class) - registerIdentifiedCallStack('dir', String.class) + this.allowedClosureSteps = closureSteps + this.allowedParameterizedClosureSteps = parameterizedClosureSteps + + allowedClosureSteps.forEach { it -> registerClosureStep(it)} + allowedParameterizedClosureSteps.entrySet().forEach { entry -> registerParameterizedClosureStep(entry.key, entry.value)} } - void runCall(String name, Integer idx = 0, Boolean doClear = true) { - CallStack callStack = callStacks[name] + DeclarativePipelineHarness runClosureStep(String name, Integer idx = 0, Boolean doClear = true) { + CallStack callStack = stepCallStacks[name] if(callStack== null) { throw new RuntimeException("No such call stack ${name}") } callStack.run(idx, doClear) + return this } - public void runIdentified(String name, T value, Boolean doClear = true) { - IdentifiedCallStack callStack = identifiedCallStacks[name] + public DeclarativePipelineHarness runParameterizedClosureStep(String name, T value, Boolean doClear = true) { + ParameterizedCallStack callStack = parameterizedStepCallStacks[name] if(callStack== null) {'pipeline' throw new RuntimeException("No such call stack ${name}") } - callStack.run(value, doClear) + this.lastClosureStepParameter = callStack.run(value, doClear) + return this } - Object runIdentifiedByIndex(String name, Integer idx = 0, Boolean doClear = true) { - IdentifiedCallStack callStack = identifiedCallStacks[name] + DeclarativePipelineHarness runParameterizedClosureStepByIndex(String name, Integer idx = 0, Boolean doClear = true) { + ParameterizedCallStack callStack = parameterizedStepCallStacks[name] if(callStack== null) { throw new RuntimeException("No such call stack ${name}") } - return callStack.run(idx, doClear) + this.lastClosureStepParameter = callStack.run(idx, doClear) + return this + } + + DeclarativePipelineHarness runStage(String name) { + return this + .runClosureStep('pipeline') + .runClosureStep('stages') + .runParameterizedClosureStep('stage', name) } - void registerCallStack(String name) { - callStacks.put(name, new CallStack(name, base.helper)) + DeclarativePipelineHarness runParallelSubStage(String parentName, String childName) { + return this + .runClosureStep('pipeline') + .runClosureStep('stages') + .runParameterizedClosureStep('stage', parentName) + .runClosureStep('parallel') + .runParameterizedClosureStep('stage', childName) } - public void registerIdentifiedCallStack(String name, Class clazz) { - identifiedCallStacks.put(name, new IdentifiedCallStack(name, base.helper, clazz)) + Object getLastClosureStepParameterValue() { + return lastClosureStepParameter + } + + void registerClosureStep(String name) { + stepCallStacks.put(name, new CallStack(name, base.helper)) + } + + public void registerParameterizedClosureStep(String name, Class clazz) { + parameterizedStepCallStacks.put(name, new ParameterizedCallStack(name, base.helper, clazz)) } void clearAllCalls() { base.helper.clearCallStack() - callStacks.values().forEach { it.clearCalls() } - identifiedCallStacks.values().forEach { it.clearCalls() } + stepCallStacks.values().forEach { it.clearCalls() } + parameterizedStepCallStacks.values().forEach { it.clearCalls() } } @@ -117,14 +109,15 @@ class DeclarativePipelineHarness { } } - class IdentifiedCallStack { + class ParameterizedCallStack { String name PipelineTestHelper helper Class clazz List values = [] List calls = [] - IdentifiedCallStack(name, helper, clazz) { + + ParameterizedCallStack(name, helper, clazz) { this.name = name this.helper = helper this.clazz = clazz diff --git a/src/test/groovy/com/lesfurets/jenkins/TestBaseDeclarativePipelineTest.groovy b/src/test/groovy/com/lesfurets/jenkins/TestBaseDeclarativePipelineTest.groovy new file mode 100644 index 00000000..70d54309 --- /dev/null +++ b/src/test/groovy/com/lesfurets/jenkins/TestBaseDeclarativePipelineTest.groovy @@ -0,0 +1,53 @@ +package com.lesfurets.jenkins + +import static org.assertj.core.api.Assertions.assertThat; + +import com.lesfurets.jenkins.unit.BaseDeclarativePipelineTest +import org.junit.Before +import org.junit.Test + +class TestBaseDeclarativePipelineTest extends BaseDeclarativePipelineTest { + @Override + String jenkinsfileName() { + return "src/test/jenkins/job/declarative.jenkins" + } + + @Override + @Before + void setUp() { + super.setUp() + } + + @Test + void runAStage() { + super.jenkinsfile.run() + harness + .runStage('init') + .runClosureStep('steps') + + def call = helper.callStack.find { it.methodName == 'sh' } + assertThat(call.args.toList()).isEqualTo(['echo \'hello world\'']) + } + + @Test + void runASubStage1() { + super.jenkinsfile.run() + harness + .runParallelSubStage('parallel build', 'build 1') + .runClosureStep('steps') + + def call = helper.callStack.find { it.methodName == 'sh' } + assertThat(call.args.toList()).isEqualTo(['echo \'hello build 1\'']) + } + + @Test + void runASubStage2() { + super.jenkinsfile.run() + harness + .runParallelSubStage('parallel build', 'build 2') + .runClosureStep('steps') + + def call = helper.callStack.find { it.methodName == 'sh' } + assertThat(call.args.toList()).isEqualTo(['echo \'hello build 2\'']) + } +} diff --git a/src/test/jenkins/job/declarative.jenkins b/src/test/jenkins/job/declarative.jenkins new file mode 100644 index 00000000..6b915728 --- /dev/null +++ b/src/test/jenkins/job/declarative.jenkins @@ -0,0 +1,23 @@ +pipeline { + stages { + stage("init") { + steps { + sh "echo 'hello world'" + } + } + stage("parallel build") { + parallel { + stage("build 1") { + steps { + sh "echo 'hello build 1'" + } + } + stage("build 2") { + steps { + sh "echo 'hello build 2'" + } + } + } + } + } +} \ No newline at end of file