Skip to content

Commit

Permalink
fix(Jenkinsfile): Fix jenkins pipline for CI
Browse files Browse the repository at this point in the history 
It appears that the current stash and unstash has some issues with asynchronous behaviour.
For example an unstash occurs on a node in a working directory that is different than what is expected when running a test.
It also appears that some directories are not being cleaned.

The following commit makes a number of changes to fix that:

- makes jenkinsfile declarative as this is better supported
- run all tests on node before releasing to fix any shared workspace problems
- Cleanup function names and steps to make it more readable
- Add timeouts to overall process and timeout per node after it starts
- Handle errors if unstash fails only stop the node
- Allow a timeout/stop to exit the whole set of tests
  • Loading branch information
@MrKevinWeiss
MrKevinWeiss committed May 18, 2020
1 parent a60786e commit a72ab97
Showing 1 changed file with 113 additions and 106 deletions.
219 changes: 113 additions & 106 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,64 @@
def nodes = nodesByLabel('HIL')
def boards = []
def tests = []
def nodeMap = [:]

def triggers = []

if ("${env.BRANCH_NAME}" == 'nightly') {
// build master with latest RIOT daily at 1:00 AM
triggers << parameterizedCron('0 1 * * * % HIL_RIOT_VERSION=master')
pipeline {
agent { label 'master' }
options {
// If the whole process takes more than x hours then exit
// This must be longer since multiple jobs can be started but waiting on nodes to complete
timeout(time: 3, unit: 'HOURS')
// Failing fast allows the nodes to be interrupted as some steps can take a while
parallelsAlwaysFailFast()
}
parameters {
choice(name: 'HIL_RIOT_VERSION', choices: ['submodule', 'master', 'pull'], description: 'The RIOT branch or PR to test.')
string(name: 'HIL_RIOT_PULL', defaultValue: '0', description: 'RIOT pull request number')
}
triggers {
parameterizedCron('0 1 * * * % HIL_RIOT_VERSION=master')
}
stages {
stage('setup') {
steps {
stepClone()
stash name: 'sources'
}
}
stage('node test') {
steps {
runParallel items: nodes.collect { "${it}" }
}
}
stage('Notify') {
steps {
emailext (
body: '''${SCRIPT, template="groovy-html.template"}''',
mimeType: 'text/html',
subject: "${currentBuild.fullDisplayName}",
from: '[email protected]',
to: '${DEFAULT_RECIPIENTS}',
replyTo: '${DEFAULT_RECIPIENTS}'
)
}
}
}
}

properties([
parameters([
choice(name: 'HIL_RIOT_VERSION', choices: ['submodule', 'master', 'pull'], description: 'The RIOT branch or PR to test.'),
string(name: 'HIL_RIOT_PULL', defaultValue: '0', description: 'RIOT pull request number')
]),
pipelineTriggers(triggers)
])
def runParallel(args) {
parallel args.items.collectEntries { name -> [ "${name}": {

node (name) {
stage("${name}") {
// We want to timeout a node if it doesn't respond
// The timeout should only start once it is acquired
timeout(time: 60, unit: 'MINUTES') {
script {
stepRunNodeTests()
}
}
}
}
}]}
}

def stepClone()
{
Expand Down Expand Up @@ -56,118 +98,83 @@ def stepClone()
}
}

def stepPrintEnv(board, test)
def stepRunNodeTests()
{
sh 'dist/tools/ci/print_environment.sh'
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
def tests = []
stage( "${env.BOARD} setup"){
stepPrepareNodeWorkingDir()
tests = stepDiscoverTests()
}
for (int i=0; i < tests.size(); i++) {
stage("${tests[i]}") {
def timeout_stop_exc = null
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE', catchInterruptions: false) {
stepPrintEnv()
stepReset(tests[i])
stepMake(tests[i])
stepFlash(tests[i])
stepTest(tests[i])
stepArchiveTestResults(tests[i])
}
}
}
}
}

def stepPrepareWorkingDir()
def stepPrepareNodeWorkingDir()
{
deleteDir()
unstash name: 'sources'
sh 'pwd'
sh 'ls -alh'
}

def stepReset(board, test)
{
sh "python3 -m bph_pal --philip_reset"
sh "make -C ${test} reset"
def stepDiscoverTests() {
return sh(returnStdout: true,
script: """
for dir in \$(find tests -maxdepth 1 -mindepth 1 -type d); do
[ -d \$dir/tests ] && { echo \$dir ; } || true
done
""").tokenize()
}

def stepFlash(board, test)
def stepPrintEnv()
{
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
sh "make -C ${test} flash"
}
sh 'dist/tools/ci/print_environment.sh'
}

def stepTests(board, test)
def stepReset(test)
{
def test_name = test.replaceAll('/', '_')
sh "make -C ${test} robot-clean || true"
catchError(buildResult: 'UNSTABLE', stageResult: 'SUCCESS') {
sh "make -C ${test} robot-test"
}
sh "make -C ${test} robot-html || true"

archiveArtifacts artifacts: "build/robot/${board}/${test_name}/*.xml"
archiveArtifacts artifacts: "build/robot/${board}/${test_name}/*.html"
junit "build/robot/${board}/${test_name}/xunit.xml"
}

// function to return steps per board
def parallelSteps (board, test) {
return {
node (board) {
catchError() {
stepPrintEnv(board, test)
stepReset(board, test)
stepFlash(board, test)
stepTests(board, test)
}
}
}
}

// detect connected boards and available tests
stage ("setup") {
node ("master") {
stepClone()
stash name: 'sources'
// discover test applications
tests = sh(returnStdout: true,
script: """
for dir in \$(find tests -maxdepth 1 -mindepth 1 -type d); do
[ -d \$dir/tests ] && { echo \$dir ; } || true
done
""").tokenize()
echo "run TESTS: " + tests.join(",")
// discover available boards
for (int i=0; i<nodes.size(); ++i) {
def nodeName = nodes[i];
node (nodeName) {
boards.push(env.BOARD)
}
}
boards.unique()
echo "use BOARDS: " + boards.join(",")
}
sh "python3 -m bph_pal --philip_reset"
sh "make -C ${test} reset"
}

for (int i=0; i<nodes.size(); ++i) {
def nodeName = nodes[i];
nodeMap[nodeName] = {
node {
stepPrepareWorkingDir()
}
}
def stepMake(test)
{
sh "make -C ${test}"
}

stage ("worker setup") {
parallel (nodeMap)
def stepFlash(test)
{
sh "make -C ${test} flash-only"
}

// create a stage per test with one step per board
for(int i=0; i < tests.size(); i++) {
test = tests[i].trim()
stage(test) {
parallel (
boards.collectEntries {
["${it}" : parallelSteps(it, test)]
}
)
def stepTest(test)
{
def test_name = test.replaceAll('/', '_')
sh "make -C ${test} robot-clean || true"
// We don't want to stop running other tests since the robot-test is allowed to fail
catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE', catchInterruptions: false) {
sh "make -C ${test} robot-test"
}
}

stage('Notify') {
node("master") {
def jobName = currentBuild.fullDisplayName
emailext (
body: '''${SCRIPT, template="groovy-html.template"}''',
mimeType: 'text/html',
subject: "${jobName}",
from: '[email protected]',
to: '${DEFAULT_RECIPIENTS}',
replyTo: '${DEFAULT_RECIPIENTS}'
)
}
def stepArchiveTestResults(test)
{
def test_name = test.replaceAll('/', '_')
sh "make -C ${test} robot-html || true"
archiveArtifacts artifacts: "build/robot/${env.BOARD}/${test_name}/*.xml"
archiveArtifacts artifacts: "build/robot/${env.BOARD}/${test_name}/*.html"
junit "build/robot/${env.BOARD}/${test_name}/xunit.xml"
}

0 comments on commit a72ab97

Please sign in to comment.